From 171a22878300de6c2e188af7b6d3be045d804bb0 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Thu, 2 Apr 2026 15:23:56 +0200 Subject: [PATCH 01/21] update --- src/checkup/materializers/console.py | 58 +++++++++++++++------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/checkup/materializers/console.py b/src/checkup/materializers/console.py index 1ccc353..03f0e2d 100644 --- a/src/checkup/materializers/console.py +++ b/src/checkup/materializers/console.py @@ -11,39 +11,43 @@ class ConsoleMaterializer(Materializer): """Output metrics to console. Outputs a rich table with metric details. + Optionally groups metrics by tag values. """ - group_tag_1: str - group_tag_2: str + group_tag_1: str | None = None + group_tag_2: str | None = None def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: - """Print metrics to console as a rich table, grouped by tags.""" + """Print metrics to console as a rich table, optionally grouped by tags.""" filtered = self._filter_metrics(metrics, direct_metric_names) - console = Console() - groups = group_metrics_by_tags(filtered, self.group_tag_1, self.group_tag_2) - - # Create a table for each group - for (tag1_value, tag2_value), group_metrics in sorted(groups.items()): - table = Table( - title=f"{self.group_tag_1}: {tag1_value} | {self.group_tag_2}: {tag2_value}" + if self.group_tag_1 and self.group_tag_2: + groups = group_metrics_by_tags(filtered, self.group_tag_1, self.group_tag_2) + for (tag1_value, tag2_value), group_metrics in sorted(groups.items()): + title = f"{self.group_tag_1}: {tag1_value} | {self.group_tag_2}: {tag2_value}" + self._print_table(console, group_metrics, title) + console.print() + else: + self._print_table(console, filtered, "Metrics") + + def _print_table(self, console: Console, metrics: list[Metric], title: str) -> None: + """Print a metrics table.""" + table = Table(title=title) + + table.add_column("Name", style="cyan", no_wrap=True) + table.add_column("Description", style="dim") + table.add_column("Value", justify="right", style="green") + table.add_column("Unit", style="yellow") + table.add_column("Diagnostics", style="red") + + for metric in metrics: + table.add_row( + metric.name, + metric.description, + str(metric.value) if metric.value is not None else "", + metric.unit, + metric.diagnostic, ) - table.add_column("Name", style="cyan", no_wrap=True) - table.add_column("Description", style="dim") - table.add_column("Value", justify="right", style="green") - table.add_column("Unit", style="yellow") - table.add_column("Diagnostics", style="red") - - for metric in group_metrics: - table.add_row( - metric.name, - metric.description, - str(metric.value) if metric.value is not None else "", - metric.unit, - metric.diagnostic, - ) - - console.print(table) - console.print() # Add spacing between tables + console.print(table) From c7310b957a1f1f82efdab82412e33f698f006148 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Fri, 3 Apr 2026 14:53:30 +0200 Subject: [PATCH 02/21] add adr --- ...04.02-metric-configuration-and-pickling.md | 227 ++++++++++++++++++ plugins/checkup-airflow/pyproject.toml | 3 + plugins/checkup-bitbucket/pyproject.toml | 3 + plugins/checkup-conveyor/pyproject.toml | 8 + plugins/checkup-dbt/pyproject.toml | 27 +++ plugins/checkup-git/pyproject.toml | 7 + plugins/checkup-github/pyproject.toml | 3 + plugins/checkup-gitlab/pyproject.toml | 3 + plugins/checkup-python/pyproject.toml | 4 + .../src/checkup_python/metrics/__init__.py | 3 +- src/checkup/cli/__init__.py | 27 +++ src/checkup/cli/commands/__init__.py | 17 ++ src/checkup/cli/commands/check.py | 48 ++++ src/checkup/cli/commands/config.py | 23 ++ src/checkup/cli/commands/init.py | 23 ++ src/checkup/cli/commands/run.py | 58 +++++ src/checkup/cli/commands/schema.py | 28 +++ src/checkup/cli/utils.py | 69 ++++++ 18 files changed, 580 insertions(+), 1 deletion(-) create mode 100644 docs/adr/2026.04.02-metric-configuration-and-pickling.md create mode 100644 src/checkup/cli/__init__.py create mode 100644 src/checkup/cli/commands/__init__.py create mode 100644 src/checkup/cli/commands/check.py create mode 100644 src/checkup/cli/commands/config.py create mode 100644 src/checkup/cli/commands/init.py create mode 100644 src/checkup/cli/commands/run.py create mode 100644 src/checkup/cli/commands/schema.py create mode 100644 src/checkup/cli/utils.py diff --git a/docs/adr/2026.04.02-metric-configuration-and-pickling.md b/docs/adr/2026.04.02-metric-configuration-and-pickling.md new file mode 100644 index 0000000..cd78295 --- /dev/null +++ b/docs/adr/2026.04.02-metric-configuration-and-pickling.md @@ -0,0 +1,227 @@ +# Metric Configuration and Pickling + +* Status: `proposed` +* Deciders: checkup team +* Proposal date: 02/04/2026 +* Decision date: 02/04/2026 + +## Context and problem statement + +Checkup currently uses a class-based design for metrics where configuration is done via subclassing: + +```python +class MyMetric(Metric): + name = "my_metric" + threshold: int = 10 # Configurable field + + def calculate(self, context, metrics): + ... +``` + +With the introduction of CLI and configuration files (`checkup.yaml`), we need to configure metrics dynamically at runtime: + +```yaml +metrics: + - name: my_metric + threshold: 20 +``` + +**The problem:** `CheckHub.measure()` uses `ProcessPoolExecutor` to parallelize execution across provider sets. This requires pickling all data sent to worker processes. Dynamically created metric subclasses (e.g., via Pydantic's `create_model`) cannot be pickled because pickle relies on importing classes by module path. + +**NOTE: Design inconsistency:** Currently, providers are configured as instances (`GitProvider(repo_path="...")`), but metrics are configured as classes by subclassing. Note that this was done by design because with CheckHub and the batch method, metrics are calculated multiple times (per provider set) and hence need to be instantiated per provider set. + +## Considered options + +1. **Pass configuration dict separately**: Add `metric_configs` parameter to CheckHub constructor, merge at instantiation time. + +2. **Register dynamic classes for pickling**: Create configured subclasses and register them in `sys.modules` so pickle can find them. + +3. **Use ThreadPoolExecutor**: Replace `ProcessPoolExecutor` with `ThreadPoolExecutor`. No pickling needed for dynamic classes. + +4. **Switch metrics to instance-based design**: Align with providers by passing metric instances instead of classes. + +5. **Use cloudpickle**: Use `cloudpickle` instead of standard `pickle` for serialization of dynamic classes. + +## Option analysis + +### Option 1: Configuration dict + +```python +CheckHub(metric_configs={"my_metric": {"threshold": 20}}) + .with_metrics([MyMetric]) + .measure() +``` + +**Pros:** +- Works with ProcessPoolExecutor +- Simple implementation + +**Cons:** +- Adds parameter to CheckHub API, shittifying the clean API we have now +- Configuration separate from metric +- Two ways to configure metrics, subclassing and dict + +### Option 2: Register dynamic classes + +```python +new_cls.__module__ = "checkup._dynamic_metrics" +setattr(sys.modules[module_name], class_name, new_cls) +``` + +**Pros:** +- Clean API, classes work normally +- Consistent with subclassing pattern + +**Cons:** +- Hidden global state +- Hacky + +### Option 3: ThreadPoolExecutor + +```python +# In CheckHub.measure() +with ThreadPoolExecutor(max_workers=workers) as executor: + ... +``` + +**Pros:** +- Simple, no API changes +- Dynamic classes work without modification + +**Cons:** +- Loses true parallelism (GIL) +- Providers using system commands (git, dbt) have threading issues + +**Why we use ProcessPoolExecutor:** Some providers (e.g., dbt, git) shell out to system commands that don't work reliably in threaded environments. Process isolation avoids these issues. This rules out ThreadPoolExecutor as a solution. + +### Option 4: Instance-based metrics + +Two sub-options for handling the fact that metrics are calculated/instantiated per provider set: + +#### Option 4a: Extract config, re-instantiate per provider set + +User passes instances of the `Metric` class, but internally we extract the class and config, and instantiate fresh per provider set. + +```python +CheckHub() + .with_metrics([MetricA(), MetricB(threshold=20)]) + .with_providers([set1, set2]) + .measure() + +# Extract class and config from instance +class CheckHub: + def with_metrics(self, metrics: list[Metric]) -> "CheckHub": + self._metric_templates = [] + for m in metrics: + metric_cls = type(m) + config = {k: getattr(m, k) for k in m.model_fields if k not in {'value', 'diagnostic', 'tags'}} + self._metric_templates.append((metric_cls, config)) + return self + +# In worker: instantiate fresh metric instance +for metric_cls, config in metric_templates: + metric = metric_cls(**config) + metric.calculate(context, calculated) +``` + +**Pros:** +- Clean API +- No pickling issues + +**Cons:** +- Slightly magical (instance becomes template) + +#### Option 4b: Separate Metric (blueprint) from Measurement (result) + +Split the current `Metric` class into two concepts: +- **Metric**: The blueprint/definition - immutable, holds config and calculate logic +- **Measurement**: The result - holds the value, tags, diagnostic + +```python +# Definition (immutable, picklable) +class Metric: + name: ClassVar[str] + threshold: int = 10 + + def calculate(self, context, metrics) -> Measurement: + value = ... + return Measurement(metric=self, value=value, tags={...}) + +# Result +class Measurement: + metric: Metric + value: Any + tags: dict + diagnostic: str + +# Usage +CheckHub() + .with_metrics([MetricA(), MetricB(threshold=20)]) + .measure() # Returns list[Measurement] +``` + +**Pros:** +- Clean separation of concerns +- Metric instances are immutable → naturally picklable +- No re-instantiation tricks needed + +**Cons:** +- Larger refactor +- Breaking API change + +### Option 5: cloudpickle + +Use `cloudpickle` to serialize dynamic classes before sending to ProcessPoolExecutor. + +A Claude-tested example of how it could look like: + +```python +import cloudpickle +from concurrent.futures import ProcessPoolExecutor + +# Pre-serialize dynamic class with cloudpickle +serialized_metrics = [cloudpickle.dumps(m) for m in metrics] + +def worker(serialized_metrics_bytes): + # Deserialize in worker process + metrics = [cloudpickle.loads(b) for b in serialized_metrics_bytes] + # ... calculate metrics ... + +with ProcessPoolExecutor() as executor: + executor.submit(worker, serialized_metrics) +``` + +**Pros:** +- Dynamic classes work + +**Cons:** +- Adds dependency +- Slightly more complex serialization + +## Chosen option + +We choose **Option 4b (Separate Metric and Measurement)** as using instances to define metrics provides a more intuitive API using constructor arguments instead of subclassing. Splitting the existing metric concept into Metric and Measurement also provides a clear separation of concerns between the definition of a metric and the result of a measurement, preferred over the ambiguity of **Option 4a**. +This also feels more consistent with how providers are configured as instances. + +Because we choose to go with using instances to configure metrics, all other options can be ruled out on the basis that they rely on subclassing. + +## Consequences + +**Breaking changes:** +- `CheckHub.measure()` returns `list[Measurement]` instead of `MeasurementResult` with `list[Metric]` +- All existing metric implementations need to be updated +- All plugins need migration + +## More information + +**Why classes for metrics originally?** +- Class-level metadata (`name`, `description`, `unit`) +- Dependency graph built from class relationships +- Multiple instantiations per provider set +- Subclassing pattern is actually pretty clean for code-defined config + +**Why ProcessPoolExecutor is required:** +- Providers like dbt and git shell out to system commands +- These system commands don't work reliably in threaded environments +- Process isolation provides clean separation and avoids threading issues +- This constraint eliminates Option 3 (ThreadPoolExecutor) as viable diff --git a/plugins/checkup-airflow/pyproject.toml b/plugins/checkup-airflow/pyproject.toml index 364d26b..cff6a8f 100644 --- a/plugins/checkup-airflow/pyproject.toml +++ b/plugins/checkup-airflow/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +airflow = "checkup_airflow:AirflowProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-bitbucket/pyproject.toml b/plugins/checkup-bitbucket/pyproject.toml index 5087151..47e5c08 100644 --- a/plugins/checkup-bitbucket/pyproject.toml +++ b/plugins/checkup-bitbucket/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +bitbucket = "checkup_bitbucket:BitbucketProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-conveyor/pyproject.toml b/plugins/checkup-conveyor/pyproject.toml index b734070..7293e6d 100644 --- a/plugins/checkup-conveyor/pyproject.toml +++ b/plugins/checkup-conveyor/pyproject.toml @@ -12,6 +12,14 @@ dependencies = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +conveyor = "checkup_conveyor:ConveyorProvider" + +[project.entry-points."checkup.metrics"] +conveyor_last_deployment_time = "checkup_conveyor.conveyor_metric:ConveyorLastDeploymentTime" +conveyor_is_dirty_deployment = "checkup_conveyor.conveyor_metric:ConveyorIsDirtyDeployment" +conveyor_last_run_status = "checkup_conveyor.conveyor_metric:ConveyorLastRunStatus" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-dbt/pyproject.toml b/plugins/checkup-dbt/pyproject.toml index aceda83..6656c70 100644 --- a/plugins/checkup-dbt/pyproject.toml +++ b/plugins/checkup-dbt/pyproject.toml @@ -18,6 +18,33 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +dbt = "checkup_dbt:DbtManifestProvider" + +[project.entry-points."checkup.metrics"] +dbt_models = "checkup_dbt:DbtModelsMetric" +dbt_columns = "checkup_dbt:DbtColumnsMetric" +dbt_tests = "checkup_dbt:DbtTestsMetric" +dbt_models_with_description = "checkup_dbt:DbtModelsWithDescriptionMetric" +dbt_models_without_description = "checkup_dbt:DbtModelsWithoutDescriptionMetric" +dbt_columns_with_description = "checkup_dbt:DbtColumnsWithDescriptionMetric" +dbt_columns_without_description = "checkup_dbt:DbtColumnsWithoutDescriptionMetric" +dbt_unit_tests = "checkup_dbt:DbtUnitTestsMetric" +dbt_data_tests = "checkup_dbt:DbtDataTestsMetric" +dbt_column_tests = "checkup_dbt:DbtColumnTestsMetric" +dbt_tested_columns = "checkup_dbt:DbtTestedColumnsMetric" +dbt_column_test_coverage = "checkup_dbt:DbtColumnTestCoverageMetric" +dbt_output_models = "checkup_dbt:DbtOutputModelsMetric" +dbt_output_models_with_description = "checkup_dbt:DbtOutputModelsWithDescriptionMetric" +dbt_output_models_without_description = "checkup_dbt:DbtOutputModelsWithoutDescriptionMetric" +dbt_output_models_without_contracts = "checkup_dbt:DbtOutputModelsWithoutContractsMetric" +dbt_output_columns_without_data_type = "checkup_dbt:DbtOutputColumnsWithoutDataTypeMetric" +dbt_flagged_packages = "checkup_dbt:DbtFlaggedPackagesMetric" +dbt_profile_host = "checkup_dbt:DbtProfileHostMetric" +dbt_models_not_adhering_to_naming_convention = "checkup_dbt:DbtModelsNotAdheringToNamingConventionMetric" +dbt_supported_version = "checkup_dbt:DbtSupportedVersionMetric" +dbt_version = "checkup_dbt:DbtVersionMetric" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-git/pyproject.toml b/plugins/checkup-git/pyproject.toml index dccc8d5..37e2bce 100644 --- a/plugins/checkup-git/pyproject.toml +++ b/plugins/checkup-git/pyproject.toml @@ -16,6 +16,13 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +git = "checkup_git:GitProvider" + +[project.entry-points."checkup.metrics"] +git_days_since_last_update = "checkup_git:GitDaysSinceLastUpdateMetric" +git_tracked_file_count = "checkup_git:GitTrackedFileCountMetric" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-github/pyproject.toml b/plugins/checkup-github/pyproject.toml index f7dd520..15bd95e 100644 --- a/plugins/checkup-github/pyproject.toml +++ b/plugins/checkup-github/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +github = "checkup_github:GitHubProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-gitlab/pyproject.toml b/plugins/checkup-gitlab/pyproject.toml index bf7e948..1fe49f9 100644 --- a/plugins/checkup-gitlab/pyproject.toml +++ b/plugins/checkup-gitlab/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +gitlab = "checkup_gitlab:GitLabProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-python/pyproject.toml b/plugins/checkup-python/pyproject.toml index cb8faba..8c40c9d 100644 --- a/plugins/checkup-python/pyproject.toml +++ b/plugins/checkup-python/pyproject.toml @@ -11,6 +11,10 @@ dependencies = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.metrics"] +python_version = "checkup_python.metrics:PythonVersionMetric" +python_version_check = "checkup_python.metrics:PythonVersionCheckMetric" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-python/src/checkup_python/metrics/__init__.py b/plugins/checkup-python/src/checkup_python/metrics/__init__.py index 85b2011..9e86bc1 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/__init__.py +++ b/plugins/checkup-python/src/checkup_python/metrics/__init__.py @@ -1,3 +1,4 @@ from checkup_python.metrics.version import PythonVersionMetric +from checkup_python.metrics.version_check import PythonVersionCheckMetric -__all__ = ["PythonVersionMetric"] +__all__ = ["PythonVersionMetric", "PythonVersionCheckMetric"] diff --git a/src/checkup/cli/__init__.py b/src/checkup/cli/__init__.py new file mode 100644 index 0000000..6b3be8c --- /dev/null +++ b/src/checkup/cli/__init__.py @@ -0,0 +1,27 @@ +""" +Checkup CLI application. +""" + +import typer + +from checkup.cli.commands import check, config, init, run, schema + +app = typer.Typer( + name="checkup", + help="CheckUp - Computational governance framework for measuring data product health", + no_args_is_help=True, +) + +app.command()(check) +app.command()(run) +app.command()(init) +app.command()(config) +app.command()(schema) + + +def main() -> None: + """ + CLI entry point. + """ + + app() diff --git a/src/checkup/cli/commands/__init__.py b/src/checkup/cli/commands/__init__.py new file mode 100644 index 0000000..55a6a66 --- /dev/null +++ b/src/checkup/cli/commands/__init__.py @@ -0,0 +1,17 @@ +""" +CLI commands. +""" + +from checkup.cli.commands.check import check +from checkup.cli.commands.config import config +from checkup.cli.commands.init import init +from checkup.cli.commands.run import run +from checkup.cli.commands.schema import schema + +__all__ = [ + "check", + "config", + "init", + "run", + "schema", +] diff --git a/src/checkup/cli/commands/check.py b/src/checkup/cli/commands/check.py new file mode 100644 index 0000000..6039ec3 --- /dev/null +++ b/src/checkup/cli/commands/check.py @@ -0,0 +1,48 @@ +""" +Check command. Run metrics locally. +""" + +import logging +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.executor import execute_checkup +from checkup.cli.utils import apply_cli_overrides +from checkup.configuration import load_config + + +def check( + config: Annotated[ + Path | None, + typer.Option("--config", "-c", help="Path to config file"), + ] = None, + tag: Annotated[ + list[str] | None, + typer.Option("--tag", "-t", help="Set tags (key=value)"), + ] = None, + provider: Annotated[ + list[str] | None, + typer.Option("--provider", "-p", help="Set providers (name or name:key=value)"), + ] = None, + metric: Annotated[ + list[str] | None, + typer.Option("--metric", "-m", help="Set metrics (name or name:key=value)"), + ] = None, + verbose: Annotated[ + bool, + typer.Option("--verbose", "-v", help="Verbose output"), + ] = False, +) -> None: + """ + Run metrics and show results. + """ + + if verbose: + logging.basicConfig(level=logging.DEBUG) + + cfg = load_config(config_path=config) + cfg = apply_cli_overrides(cfg, tag, provider, metric) + + execute_checkup(cfg, materializer="console") diff --git a/src/checkup/cli/commands/config.py b/src/checkup/cli/commands/config.py new file mode 100644 index 0000000..e27f25e --- /dev/null +++ b/src/checkup/cli/commands/config.py @@ -0,0 +1,23 @@ +""" +Config command. Modify an existing config file. +""" + +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.config_wizard import edit_config + + +def config( + config_path: Annotated[ + Path | None, + typer.Option("--config", "-c", help="Path to config file"), + ] = None, +) -> None: + """ + Modify the checkup.yaml config file. + """ + + edit_config(config_path=config_path) diff --git a/src/checkup/cli/commands/init.py b/src/checkup/cli/commands/init.py new file mode 100644 index 0000000..a224cb1 --- /dev/null +++ b/src/checkup/cli/commands/init.py @@ -0,0 +1,23 @@ +""" +Init command. Create a new config file. +""" + +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.config_wizard import create_config + + +def init( + output: Annotated[ + Path | None, + typer.Option("--output", "-o", help="Output path for config file"), + ] = None, +) -> None: + """ + Create a checkup.yaml config file. + """ + + create_config(output_path=output) diff --git a/src/checkup/cli/commands/run.py b/src/checkup/cli/commands/run.py new file mode 100644 index 0000000..1d1ca49 --- /dev/null +++ b/src/checkup/cli/commands/run.py @@ -0,0 +1,58 @@ +""" +Run command. Run metrics and materialize results. +""" + +import logging +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.executor import execute_checkup +from checkup.cli.utils import apply_cli_overrides +from checkup.configuration import load_config + + +def run( + config: Annotated[ + Path | None, + typer.Option("--config", "-c", help="Path to config file"), + ] = None, + tag: Annotated[ + list[str] | None, + typer.Option("--tag", "-t", help="Set tags (key=value)"), + ] = None, + provider: Annotated[ + list[str] | None, + typer.Option("--provider", "-p", help="Set providers (name or name:key=value)"), + ] = None, + metric: Annotated[ + list[str] | None, + typer.Option("--metric", "-m", help="Set metrics (name or name:key=value)"), + ] = None, + materializer: Annotated[ + str | None, + typer.Option( + "--materializer", help="Set materializer (type or type:key=value)" + ), + ] = None, + dry_run: Annotated[ + bool, + typer.Option("--dry-run", help="Don't materialize, just print (same as check)"), + ] = False, + verbose: Annotated[ + bool, + typer.Option("--verbose", "-v", help="Verbose output"), + ] = False, +) -> None: + """ + Run metrics and materialize results. + """ + + if verbose: + logging.basicConfig(level=logging.DEBUG) + + cfg = load_config(config_path=config) + cfg = apply_cli_overrides(cfg, tag, provider, metric) + + execute_checkup(cfg, materializer="console" if dry_run else materializer) diff --git a/src/checkup/cli/commands/schema.py b/src/checkup/cli/commands/schema.py new file mode 100644 index 0000000..7fbe768 --- /dev/null +++ b/src/checkup/cli/commands/schema.py @@ -0,0 +1,28 @@ +""" +Schema command. Generate JSON schema for checkup.yaml. +""" + +from pathlib import Path +from typing import Annotated + +import typer +from rich.console import Console + +from checkup.configuration.schema import write_schema + +console = Console() + + +def schema( + output: Annotated[ + Path | None, + typer.Option("--output", "-o", help="Output path for schema file"), + ] = None, +) -> None: + """ + Generate JSON schema for checkup.yaml. + """ + + path = output or Path.cwd() / "checkup.schema.json" + write_schema(path) + console.print(f"[green]Schema written to {path}[/green]") diff --git a/src/checkup/cli/utils.py b/src/checkup/cli/utils.py new file mode 100644 index 0000000..e2e5da4 --- /dev/null +++ b/src/checkup/cli/utils.py @@ -0,0 +1,69 @@ +""" +CLI utility functions. +""" + +from checkup.configuration import CheckupConfig, MetricConfig, ProviderConfig + + +def apply_cli_overrides( + cfg: CheckupConfig, + tags: list[str] | None, + providers: list[str] | None, + metrics: list[str] | None, +) -> CheckupConfig: + """ + Apply CLI flag overrides to config. + + When CLI arguments are provided, they replace the config file values + """ + + if tags: + new_tags = {} + for t in tags: + if "=" in t: + key, value = t.split("=", 1) + new_tags[key] = value + else: + new_tags = dict(cfg.tags) + + if providers: + new_providers = [] + for p in providers: + name, config = parse_cli_item(p) + new_providers.append(ProviderConfig(name=name, config=config)) + else: + new_providers = list(cfg.providers) + + if metrics: + new_metrics = [] + for m in metrics: + name, config = parse_cli_item(m) + new_metrics.append(MetricConfig(name=name, config=config)) + else: + new_metrics = list(cfg.metrics) + + return CheckupConfig( + tags=new_tags, + providers=new_providers, + metrics=new_metrics, + materializer=cfg.materializer, + ) + + +def parse_cli_item(item: str) -> tuple[str, dict]: + """ + Parse CLI item like 'name' or 'name:key=value,key2=value2'. + """ + + if ":" not in item: + return item, {} + + name, config_str = item.split(":", 1) + config = {} + + for pair in config_str.split(","): + if "=" in pair: + key, value = pair.split("=", 1) + config[key] = value + + return name, config From c5e9de5c830efe79156d29a67676dd28ee318ef3 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Fri, 3 Apr 2026 15:00:50 +0200 Subject: [PATCH 03/21] Keep only ADR, reset other changes Co-Authored-By: Claude Opus 4.5 --- plugins/checkup-airflow/pyproject.toml | 3 - plugins/checkup-bitbucket/pyproject.toml | 3 - plugins/checkup-conveyor/pyproject.toml | 8 --- plugins/checkup-dbt/pyproject.toml | 27 -------- plugins/checkup-git/pyproject.toml | 7 -- plugins/checkup-github/pyproject.toml | 3 - plugins/checkup-gitlab/pyproject.toml | 3 - plugins/checkup-python/pyproject.toml | 4 -- .../src/checkup_python/metrics/__init__.py | 3 +- src/checkup/cli/__init__.py | 27 -------- src/checkup/cli/commands/__init__.py | 17 ----- src/checkup/cli/commands/check.py | 48 ------------- src/checkup/cli/commands/config.py | 23 ------- src/checkup/cli/commands/init.py | 23 ------- src/checkup/cli/commands/run.py | 58 ---------------- src/checkup/cli/commands/schema.py | 28 -------- src/checkup/cli/utils.py | 69 ------------------- 17 files changed, 1 insertion(+), 353 deletions(-) delete mode 100644 src/checkup/cli/__init__.py delete mode 100644 src/checkup/cli/commands/__init__.py delete mode 100644 src/checkup/cli/commands/check.py delete mode 100644 src/checkup/cli/commands/config.py delete mode 100644 src/checkup/cli/commands/init.py delete mode 100644 src/checkup/cli/commands/run.py delete mode 100644 src/checkup/cli/commands/schema.py delete mode 100644 src/checkup/cli/utils.py diff --git a/plugins/checkup-airflow/pyproject.toml b/plugins/checkup-airflow/pyproject.toml index cff6a8f..364d26b 100644 --- a/plugins/checkup-airflow/pyproject.toml +++ b/plugins/checkup-airflow/pyproject.toml @@ -16,9 +16,6 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.providers"] -airflow = "checkup_airflow:AirflowProvider" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-bitbucket/pyproject.toml b/plugins/checkup-bitbucket/pyproject.toml index 47e5c08..5087151 100644 --- a/plugins/checkup-bitbucket/pyproject.toml +++ b/plugins/checkup-bitbucket/pyproject.toml @@ -16,9 +16,6 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.providers"] -bitbucket = "checkup_bitbucket:BitbucketProvider" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-conveyor/pyproject.toml b/plugins/checkup-conveyor/pyproject.toml index 7293e6d..b734070 100644 --- a/plugins/checkup-conveyor/pyproject.toml +++ b/plugins/checkup-conveyor/pyproject.toml @@ -12,14 +12,6 @@ dependencies = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.providers"] -conveyor = "checkup_conveyor:ConveyorProvider" - -[project.entry-points."checkup.metrics"] -conveyor_last_deployment_time = "checkup_conveyor.conveyor_metric:ConveyorLastDeploymentTime" -conveyor_is_dirty_deployment = "checkup_conveyor.conveyor_metric:ConveyorIsDirtyDeployment" -conveyor_last_run_status = "checkup_conveyor.conveyor_metric:ConveyorLastRunStatus" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-dbt/pyproject.toml b/plugins/checkup-dbt/pyproject.toml index 6656c70..aceda83 100644 --- a/plugins/checkup-dbt/pyproject.toml +++ b/plugins/checkup-dbt/pyproject.toml @@ -18,33 +18,6 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.providers"] -dbt = "checkup_dbt:DbtManifestProvider" - -[project.entry-points."checkup.metrics"] -dbt_models = "checkup_dbt:DbtModelsMetric" -dbt_columns = "checkup_dbt:DbtColumnsMetric" -dbt_tests = "checkup_dbt:DbtTestsMetric" -dbt_models_with_description = "checkup_dbt:DbtModelsWithDescriptionMetric" -dbt_models_without_description = "checkup_dbt:DbtModelsWithoutDescriptionMetric" -dbt_columns_with_description = "checkup_dbt:DbtColumnsWithDescriptionMetric" -dbt_columns_without_description = "checkup_dbt:DbtColumnsWithoutDescriptionMetric" -dbt_unit_tests = "checkup_dbt:DbtUnitTestsMetric" -dbt_data_tests = "checkup_dbt:DbtDataTestsMetric" -dbt_column_tests = "checkup_dbt:DbtColumnTestsMetric" -dbt_tested_columns = "checkup_dbt:DbtTestedColumnsMetric" -dbt_column_test_coverage = "checkup_dbt:DbtColumnTestCoverageMetric" -dbt_output_models = "checkup_dbt:DbtOutputModelsMetric" -dbt_output_models_with_description = "checkup_dbt:DbtOutputModelsWithDescriptionMetric" -dbt_output_models_without_description = "checkup_dbt:DbtOutputModelsWithoutDescriptionMetric" -dbt_output_models_without_contracts = "checkup_dbt:DbtOutputModelsWithoutContractsMetric" -dbt_output_columns_without_data_type = "checkup_dbt:DbtOutputColumnsWithoutDataTypeMetric" -dbt_flagged_packages = "checkup_dbt:DbtFlaggedPackagesMetric" -dbt_profile_host = "checkup_dbt:DbtProfileHostMetric" -dbt_models_not_adhering_to_naming_convention = "checkup_dbt:DbtModelsNotAdheringToNamingConventionMetric" -dbt_supported_version = "checkup_dbt:DbtSupportedVersionMetric" -dbt_version = "checkup_dbt:DbtVersionMetric" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-git/pyproject.toml b/plugins/checkup-git/pyproject.toml index 37e2bce..dccc8d5 100644 --- a/plugins/checkup-git/pyproject.toml +++ b/plugins/checkup-git/pyproject.toml @@ -16,13 +16,6 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.providers"] -git = "checkup_git:GitProvider" - -[project.entry-points."checkup.metrics"] -git_days_since_last_update = "checkup_git:GitDaysSinceLastUpdateMetric" -git_tracked_file_count = "checkup_git:GitTrackedFileCountMetric" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-github/pyproject.toml b/plugins/checkup-github/pyproject.toml index 15bd95e..f7dd520 100644 --- a/plugins/checkup-github/pyproject.toml +++ b/plugins/checkup-github/pyproject.toml @@ -16,9 +16,6 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.providers"] -github = "checkup_github:GitHubProvider" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-gitlab/pyproject.toml b/plugins/checkup-gitlab/pyproject.toml index 1fe49f9..bf7e948 100644 --- a/plugins/checkup-gitlab/pyproject.toml +++ b/plugins/checkup-gitlab/pyproject.toml @@ -16,9 +16,6 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.providers"] -gitlab = "checkup_gitlab:GitLabProvider" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-python/pyproject.toml b/plugins/checkup-python/pyproject.toml index 8c40c9d..cb8faba 100644 --- a/plugins/checkup-python/pyproject.toml +++ b/plugins/checkup-python/pyproject.toml @@ -11,10 +11,6 @@ dependencies = [ [tool.uv.sources] checkup = { workspace = true } -[project.entry-points."checkup.metrics"] -python_version = "checkup_python.metrics:PythonVersionMetric" -python_version_check = "checkup_python.metrics:PythonVersionCheckMetric" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-python/src/checkup_python/metrics/__init__.py b/plugins/checkup-python/src/checkup_python/metrics/__init__.py index 9e86bc1..85b2011 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/__init__.py +++ b/plugins/checkup-python/src/checkup_python/metrics/__init__.py @@ -1,4 +1,3 @@ from checkup_python.metrics.version import PythonVersionMetric -from checkup_python.metrics.version_check import PythonVersionCheckMetric -__all__ = ["PythonVersionMetric", "PythonVersionCheckMetric"] +__all__ = ["PythonVersionMetric"] diff --git a/src/checkup/cli/__init__.py b/src/checkup/cli/__init__.py deleted file mode 100644 index 6b3be8c..0000000 --- a/src/checkup/cli/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Checkup CLI application. -""" - -import typer - -from checkup.cli.commands import check, config, init, run, schema - -app = typer.Typer( - name="checkup", - help="CheckUp - Computational governance framework for measuring data product health", - no_args_is_help=True, -) - -app.command()(check) -app.command()(run) -app.command()(init) -app.command()(config) -app.command()(schema) - - -def main() -> None: - """ - CLI entry point. - """ - - app() diff --git a/src/checkup/cli/commands/__init__.py b/src/checkup/cli/commands/__init__.py deleted file mode 100644 index 55a6a66..0000000 --- a/src/checkup/cli/commands/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -CLI commands. -""" - -from checkup.cli.commands.check import check -from checkup.cli.commands.config import config -from checkup.cli.commands.init import init -from checkup.cli.commands.run import run -from checkup.cli.commands.schema import schema - -__all__ = [ - "check", - "config", - "init", - "run", - "schema", -] diff --git a/src/checkup/cli/commands/check.py b/src/checkup/cli/commands/check.py deleted file mode 100644 index 6039ec3..0000000 --- a/src/checkup/cli/commands/check.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Check command. Run metrics locally. -""" - -import logging -from pathlib import Path -from typing import Annotated - -import typer - -from checkup.cli.executor import execute_checkup -from checkup.cli.utils import apply_cli_overrides -from checkup.configuration import load_config - - -def check( - config: Annotated[ - Path | None, - typer.Option("--config", "-c", help="Path to config file"), - ] = None, - tag: Annotated[ - list[str] | None, - typer.Option("--tag", "-t", help="Set tags (key=value)"), - ] = None, - provider: Annotated[ - list[str] | None, - typer.Option("--provider", "-p", help="Set providers (name or name:key=value)"), - ] = None, - metric: Annotated[ - list[str] | None, - typer.Option("--metric", "-m", help="Set metrics (name or name:key=value)"), - ] = None, - verbose: Annotated[ - bool, - typer.Option("--verbose", "-v", help="Verbose output"), - ] = False, -) -> None: - """ - Run metrics and show results. - """ - - if verbose: - logging.basicConfig(level=logging.DEBUG) - - cfg = load_config(config_path=config) - cfg = apply_cli_overrides(cfg, tag, provider, metric) - - execute_checkup(cfg, materializer="console") diff --git a/src/checkup/cli/commands/config.py b/src/checkup/cli/commands/config.py deleted file mode 100644 index e27f25e..0000000 --- a/src/checkup/cli/commands/config.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Config command. Modify an existing config file. -""" - -from pathlib import Path -from typing import Annotated - -import typer - -from checkup.cli.config_wizard import edit_config - - -def config( - config_path: Annotated[ - Path | None, - typer.Option("--config", "-c", help="Path to config file"), - ] = None, -) -> None: - """ - Modify the checkup.yaml config file. - """ - - edit_config(config_path=config_path) diff --git a/src/checkup/cli/commands/init.py b/src/checkup/cli/commands/init.py deleted file mode 100644 index a224cb1..0000000 --- a/src/checkup/cli/commands/init.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Init command. Create a new config file. -""" - -from pathlib import Path -from typing import Annotated - -import typer - -from checkup.cli.config_wizard import create_config - - -def init( - output: Annotated[ - Path | None, - typer.Option("--output", "-o", help="Output path for config file"), - ] = None, -) -> None: - """ - Create a checkup.yaml config file. - """ - - create_config(output_path=output) diff --git a/src/checkup/cli/commands/run.py b/src/checkup/cli/commands/run.py deleted file mode 100644 index 1d1ca49..0000000 --- a/src/checkup/cli/commands/run.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Run command. Run metrics and materialize results. -""" - -import logging -from pathlib import Path -from typing import Annotated - -import typer - -from checkup.cli.executor import execute_checkup -from checkup.cli.utils import apply_cli_overrides -from checkup.configuration import load_config - - -def run( - config: Annotated[ - Path | None, - typer.Option("--config", "-c", help="Path to config file"), - ] = None, - tag: Annotated[ - list[str] | None, - typer.Option("--tag", "-t", help="Set tags (key=value)"), - ] = None, - provider: Annotated[ - list[str] | None, - typer.Option("--provider", "-p", help="Set providers (name or name:key=value)"), - ] = None, - metric: Annotated[ - list[str] | None, - typer.Option("--metric", "-m", help="Set metrics (name or name:key=value)"), - ] = None, - materializer: Annotated[ - str | None, - typer.Option( - "--materializer", help="Set materializer (type or type:key=value)" - ), - ] = None, - dry_run: Annotated[ - bool, - typer.Option("--dry-run", help="Don't materialize, just print (same as check)"), - ] = False, - verbose: Annotated[ - bool, - typer.Option("--verbose", "-v", help="Verbose output"), - ] = False, -) -> None: - """ - Run metrics and materialize results. - """ - - if verbose: - logging.basicConfig(level=logging.DEBUG) - - cfg = load_config(config_path=config) - cfg = apply_cli_overrides(cfg, tag, provider, metric) - - execute_checkup(cfg, materializer="console" if dry_run else materializer) diff --git a/src/checkup/cli/commands/schema.py b/src/checkup/cli/commands/schema.py deleted file mode 100644 index 7fbe768..0000000 --- a/src/checkup/cli/commands/schema.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Schema command. Generate JSON schema for checkup.yaml. -""" - -from pathlib import Path -from typing import Annotated - -import typer -from rich.console import Console - -from checkup.configuration.schema import write_schema - -console = Console() - - -def schema( - output: Annotated[ - Path | None, - typer.Option("--output", "-o", help="Output path for schema file"), - ] = None, -) -> None: - """ - Generate JSON schema for checkup.yaml. - """ - - path = output or Path.cwd() / "checkup.schema.json" - write_schema(path) - console.print(f"[green]Schema written to {path}[/green]") diff --git a/src/checkup/cli/utils.py b/src/checkup/cli/utils.py deleted file mode 100644 index e2e5da4..0000000 --- a/src/checkup/cli/utils.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -CLI utility functions. -""" - -from checkup.configuration import CheckupConfig, MetricConfig, ProviderConfig - - -def apply_cli_overrides( - cfg: CheckupConfig, - tags: list[str] | None, - providers: list[str] | None, - metrics: list[str] | None, -) -> CheckupConfig: - """ - Apply CLI flag overrides to config. - - When CLI arguments are provided, they replace the config file values - """ - - if tags: - new_tags = {} - for t in tags: - if "=" in t: - key, value = t.split("=", 1) - new_tags[key] = value - else: - new_tags = dict(cfg.tags) - - if providers: - new_providers = [] - for p in providers: - name, config = parse_cli_item(p) - new_providers.append(ProviderConfig(name=name, config=config)) - else: - new_providers = list(cfg.providers) - - if metrics: - new_metrics = [] - for m in metrics: - name, config = parse_cli_item(m) - new_metrics.append(MetricConfig(name=name, config=config)) - else: - new_metrics = list(cfg.metrics) - - return CheckupConfig( - tags=new_tags, - providers=new_providers, - metrics=new_metrics, - materializer=cfg.materializer, - ) - - -def parse_cli_item(item: str) -> tuple[str, dict]: - """ - Parse CLI item like 'name' or 'name:key=value,key2=value2'. - """ - - if ":" not in item: - return item, {} - - name, config_str = item.split(":", 1) - config = {} - - for pair in config_str.split(","): - if "=" in pair: - key, value = pair.split("=", 1) - config[key] = value - - return name, config From 02fb78f24b83a073fbf1e13d604c3661cbaaf21f Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Thu, 2 Apr 2026 15:23:56 +0200 Subject: [PATCH 04/21] update --- src/checkup/materializers/console.py | 58 +++++++++++++++------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/checkup/materializers/console.py b/src/checkup/materializers/console.py index 1ccc353..03f0e2d 100644 --- a/src/checkup/materializers/console.py +++ b/src/checkup/materializers/console.py @@ -11,39 +11,43 @@ class ConsoleMaterializer(Materializer): """Output metrics to console. Outputs a rich table with metric details. + Optionally groups metrics by tag values. """ - group_tag_1: str - group_tag_2: str + group_tag_1: str | None = None + group_tag_2: str | None = None def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: - """Print metrics to console as a rich table, grouped by tags.""" + """Print metrics to console as a rich table, optionally grouped by tags.""" filtered = self._filter_metrics(metrics, direct_metric_names) - console = Console() - groups = group_metrics_by_tags(filtered, self.group_tag_1, self.group_tag_2) - - # Create a table for each group - for (tag1_value, tag2_value), group_metrics in sorted(groups.items()): - table = Table( - title=f"{self.group_tag_1}: {tag1_value} | {self.group_tag_2}: {tag2_value}" + if self.group_tag_1 and self.group_tag_2: + groups = group_metrics_by_tags(filtered, self.group_tag_1, self.group_tag_2) + for (tag1_value, tag2_value), group_metrics in sorted(groups.items()): + title = f"{self.group_tag_1}: {tag1_value} | {self.group_tag_2}: {tag2_value}" + self._print_table(console, group_metrics, title) + console.print() + else: + self._print_table(console, filtered, "Metrics") + + def _print_table(self, console: Console, metrics: list[Metric], title: str) -> None: + """Print a metrics table.""" + table = Table(title=title) + + table.add_column("Name", style="cyan", no_wrap=True) + table.add_column("Description", style="dim") + table.add_column("Value", justify="right", style="green") + table.add_column("Unit", style="yellow") + table.add_column("Diagnostics", style="red") + + for metric in metrics: + table.add_row( + metric.name, + metric.description, + str(metric.value) if metric.value is not None else "", + metric.unit, + metric.diagnostic, ) - table.add_column("Name", style="cyan", no_wrap=True) - table.add_column("Description", style="dim") - table.add_column("Value", justify="right", style="green") - table.add_column("Unit", style="yellow") - table.add_column("Diagnostics", style="red") - - for metric in group_metrics: - table.add_row( - metric.name, - metric.description, - str(metric.value) if metric.value is not None else "", - metric.unit, - metric.diagnostic, - ) - - console.print(table) - console.print() # Add spacing between tables + console.print(table) From 65b98747f13e99d68ca9d4925ad638a474b7b762 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Fri, 3 Apr 2026 16:27:14 +0200 Subject: [PATCH 05/21] plugins & docs --- docs/concepts/materializers.md | 43 +++++----- docs/concepts/metrics.md | 85 +++++++++++-------- docs/getting-started/quickstart.md | 40 ++++----- docs/index.md | 9 +- docs/plugins/conveyor.md | 10 ++- docs/plugins/overview.md | 8 +- plugins/checkup-conveyor/README.md | 9 +- .../src/checkup_conveyor/conveyor_metric.py | 49 ++++++----- plugins/checkup-dbt/README.md | 16 ++-- .../src/checkup_dbt/metrics/base.py | 33 ++++--- .../metrics/quality/flagged_packages.py | 27 +++--- .../metrics/quality/naming_convention.py | 16 ++-- .../metrics/quality/profile_host.py | 23 +++-- .../metrics/quality/supported_version.py | 20 +++-- .../checkup_dbt/metrics/quality/version.py | 15 ++-- .../metrics/test/column_test_coverage.py | 20 +++-- .../metrics/test/tested_columns.py | 10 ++- plugins/checkup-git/README.md | 13 +-- .../checkup-git/src/checkup_git/metrics.py | 37 ++++---- .../src/checkup_python/metrics/version.py | 8 +- .../checkup_python/metrics/version_check.py | 12 +-- 21 files changed, 283 insertions(+), 220 deletions(-) diff --git a/docs/concepts/materializers.md b/docs/concepts/materializers.md index 9095aa8..217de86 100644 --- a/docs/concepts/materializers.md +++ b/docs/concepts/materializers.md @@ -19,7 +19,7 @@ Materializers are called on `MeasurementResult`: ```python from checkup import CheckHub, ConsoleMaterializer -result = CheckHub().with_metrics([MyMetric]).measure() +result = CheckHub().with_metrics([MyMetric()]).measure() result.materialize(ConsoleMaterializer(group_tag_1="domain", group_tag_2="project")) ``` @@ -104,7 +104,7 @@ Features: You can materialize to multiple formats: ```python -result = CheckHub().with_metrics([MyMetric]).measure() +result = CheckHub().with_metrics([MyMetric()]).measure() # Output to console result.materialize(ConsoleMaterializer(group_tag_1="domain", group_tag_2="project")) @@ -152,10 +152,11 @@ Materializers group output by metric tags: ```python class MyMetric(Metric): - def calculate(self, context, metrics): - self.value = 42 - self.tags["domain"] = "analytics" - self.tags["project"] = "dashboard" + def calculate(self, context, measurements): + return self.measurement( + value=42, + tags={"domain": "analytics", "project": "dashboard"} + ) ``` When materialized with `group_tag_1="domain"` and `group_tag_2="project"`, metrics are grouped accordingly. @@ -165,8 +166,8 @@ When materialized with `group_tag_1="domain"` and `group_tag_2="project"`, metri Create custom materializers by extending the base class: ```python -from checkup import Materializer, Metric -from pydantic import BaseModel +from checkup import Materializer, Measurement +from pathlib import Path class JSONMaterializer(Materializer): @@ -174,15 +175,15 @@ class JSONMaterializer(Materializer): output_path: Path - def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: - filtered = self._filter_metrics(metrics, direct_metric_names) + def materialize(self, measurements: list[Measurement], direct_metric_names: set[str]) -> None: + filtered = self._filter_measurements(measurements, direct_metric_names) data = [ { - "name": m.name, + "name": m.metric_name, "value": m.value, - "unit": m.unit, - "description": m.description, + "unit": m.metric_unit, + "description": m.metric_description, "diagnostic": m.diagnostic, "tags": m.tags, } @@ -196,28 +197,28 @@ class JSONMaterializer(Materializer): ## Utility Functions -CheckUp provides helper functions for grouping metrics: +CheckUp provides helper functions for grouping measurements: ```python -from checkup.materializers import group_metrics_by_tags, group_metrics_hierarchical +from checkup.materializers import group_measurements_by_tags, group_measurements_hierarchical # Flat grouping by two tags -groups = group_metrics_by_tags( - metrics, +groups = group_measurements_by_tags( + measurements, tag1="domain", tag2="project", default_value="Unknown" ) -# Returns: {("analytics", "dashboard"): [metrics...]} +# Returns: {("analytics", "dashboard"): [measurements...]} # Hierarchical grouping -hierarchy = group_metrics_hierarchical( - metrics, +hierarchy = group_measurements_hierarchical( + measurements, tag1="domain", tag2="project", default_value="Ungrouped" ) -# Returns: {"analytics": {"dashboard": [metrics...]}} +# Returns: {"analytics": {"dashboard": [measurements...]}} ``` ## Best Practices diff --git a/docs/concepts/metrics.md b/docs/concepts/metrics.md index abf5f9b..38f389b 100644 --- a/docs/concepts/metrics.md +++ b/docs/concepts/metrics.md @@ -7,7 +7,7 @@ Metrics are the core building blocks of CheckUp. They calculate values from cont Create a metric by subclassing the `Metric` base class: ```python -from checkup import Metric +from checkup import Metric, Measurement from checkup.types import Context @@ -17,10 +17,9 @@ class MyMetric(Metric): description = "Description of what this metric measures" unit = "count" - def calculate(self, context: Context, metrics: dict) -> None: + def calculate(self, context: Context, measurements: dict) -> Measurement: # Your calculation logic here - self.value = 42 - self.diagnostic = "Additional information" + return self.measurement(value=42, diagnostic="Additional information") ``` ## Required Attributes @@ -33,35 +32,43 @@ Every metric must define these class attributes: | `description` | `str` | Human-readable description | | `unit` | `str` | Unit of measurement (e.g., "count", "percent", "ms") | -## Instance Attributes +## The Measurement Class -After calculation, metrics have these instance attributes: +The `calculate` method returns a `Measurement` object containing: | Attribute | Type | Description | |-----------|------|-------------| +| `metric_name` | `str` | Name of the metric (set automatically) | +| `metric_description` | `str` | Description of the metric (set automatically) | +| `metric_unit` | `str` | Unit of measurement (set automatically) | | `value` | `Any` | The calculated metric value | | `diagnostic` | `str` | Additional diagnostic information | | `tags` | `dict` | Key-value pairs for grouping/filtering | +Use `self.measurement()` to create a `Measurement` with the metric's metadata pre-filled: + +```python +return self.measurement(value=42, diagnostic="Explanation", tags={"key": "value"}) +``` + ## The Calculate Method The `calculate` method is where you implement your metric logic: ```python -def calculate(self, context: Context, metrics: dict) -> None: +def calculate(self, context: Context, measurements: dict) -> Measurement: # context: Dict containing provider data under namespaces - # metrics: Dict mapping metric classes to calculated instances + # measurements: Dict mapping metric classes to their Measurement results # Access provider data git_data = context.get("git", {}) # Access dependency metrics - if SomeOtherMetric in metrics: - other_value = metrics[SomeOtherMetric].value + if SomeOtherMetric in measurements: + other_value = measurements[SomeOtherMetric].value - # Set the result - self.value = computed_value - self.diagnostic = "Explanation of result" + # Return the result + return self.measurement(value=computed_value, diagnostic="Explanation of result") ``` ## Dependencies @@ -74,8 +81,8 @@ class BaseMetric(Metric): description = "A base metric" unit = "count" - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 100 + def calculate(self, context: Context, measurements: dict) -> Measurement: + return self.measurement(value=100) class DerivedMetric(Metric): @@ -87,9 +94,9 @@ class DerivedMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [BaseMetric] - def calculate(self, context: Context, metrics: dict) -> None: - base_value = metrics[BaseMetric].value - self.value = base_value * 0.5 + def calculate(self, context: Context, measurements: dict) -> Measurement: + base_value = measurements[BaseMetric].value + return self.measurement(value=base_value * 0.5) ``` ## Providers @@ -114,10 +121,10 @@ class MyMetric(Metric): def providers(cls) -> list[type[Provider]]: return [MyDataProvider] - def calculate(self, context: Context, metrics: dict) -> None: + def calculate(self, context: Context, measurements: dict) -> Measurement: # Access provider data under its namespace data = context["my_data"] - self.value = len(data.get("items", [])) + return self.measurement(value=len(data.get("items", []))) ``` ## Executor Types @@ -151,7 +158,7 @@ class AsyncMetric(Metric): ## Tags -Tags allow grouping and filtering metrics: +Tags allow grouping and filtering metrics. Tags can be set when creating the measurement: ```python class TaggedMetric(Metric): @@ -159,13 +166,15 @@ class TaggedMetric(Metric): description = "A metric with tags" unit = "count" - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 42 - # Tags can be set during calculation - self.tags["domain"] = "data-platform" - self.tags["project"] = "analytics" + def calculate(self, context: Context, measurements: dict) -> Measurement: + return self.measurement( + value=42, + tags={"domain": "data-platform", "project": "analytics"} + ) ``` +Tags are also merged from `TagProvider` instances in the provider set. + Tags are used by materializers for grouping output: ```python @@ -186,7 +195,7 @@ result.materialize( ## Example: Complete Metric ```python -from checkup import Metric, Provider, ExecutorType +from checkup import Metric, Provider, ExecutorType, Measurement from checkup.types import Context @@ -200,23 +209,25 @@ class CodeCoverageMetric(Metric): def providers(cls) -> list[type[Provider]]: return [CoverageReportProvider] - def calculate(self, context: Context, metrics: dict) -> None: + def calculate(self, context: Context, measurements: dict) -> Measurement: coverage_data = context.get("coverage", {}) if not coverage_data: - self.value = None - self.diagnostic = "No coverage data available" - return + return self.measurement(value=None, diagnostic="No coverage data available") total_lines = coverage_data.get("total_lines", 0) covered_lines = coverage_data.get("covered_lines", 0) if total_lines > 0: - self.value = round((covered_lines / total_lines) * 100, 2) - self.diagnostic = f"{covered_lines}/{total_lines} lines covered" + value = round((covered_lines / total_lines) * 100, 2) + diagnostic = f"{covered_lines}/{total_lines} lines covered" else: - self.value = 0 - self.diagnostic = "No lines to cover" - - self.tags["project"] = coverage_data.get("project_name", "unknown") + value = 0 + diagnostic = "No lines to cover" + + return self.measurement( + value=value, + diagnostic=diagnostic, + tags={"project": coverage_data.get("project_name", "unknown")} + ) ``` diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index e693b1b..f23e9a3 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -4,10 +4,10 @@ This guide will help you get started with CheckUp by walking through a complete ## Creating Your First Metric -A metric is a class that calculates a value from context. Here's a simple example: +A metric is a class that calculates a value from context and returns a Measurement. Here's a simple example: ```python -from checkup import Metric +from checkup import Metric, Measurement from checkup.types import Context @@ -16,19 +16,20 @@ class FileCountMetric(Metric): description = "Number of files in the project" unit = "files" - def calculate(self, context: Context, metrics: dict) -> None: + def calculate(self, context: Context, measurements: dict) -> Measurement: # Access data from context and calculate your metric files = context.get("files", []) - self.value = len(files) - self.diagnostic = f"Found {self.value} files" + return self.measurement( + value=len(files), + diagnostic=f"Found {len(files)} files" + ) ``` Every metric must: 1. Define `name`, `description`, and `unit` class attributes 2. Implement the `calculate()` method -3. Set `self.value` with the calculated result -4. Optionally set `self.diagnostic` with additional information +3. Return a `Measurement` using `self.measurement(value=..., diagnostic=...)` ## Running Metrics with CheckHub @@ -41,7 +42,7 @@ from checkup import CheckHub, ConsoleMaterializer # Create and run the pipeline result = ( CheckHub() - .with_metrics([FileCountMetric]) + .with_metrics([FileCountMetric()]) .measure() ) @@ -84,10 +85,10 @@ class FileCountMetric(Metric): def providers(cls) -> list[type[Provider]]: return [FileProvider] - def calculate(self, context: Context, metrics: dict) -> None: + def calculate(self, context: Context, measurements: dict) -> Measurement: # Access provider data under its namespace files = context["files"]["file_list"] - self.value = len(files) + return self.measurement(value=len(files)) ``` Run with providers: @@ -97,7 +98,7 @@ from pathlib import Path result = ( CheckHub() - .with_metrics([FileCountMetric]) + .with_metrics([FileCountMetric()]) .with_providers([ [FileProvider(Path("./src"))], [FileProvider(Path("./tests"))], @@ -116,9 +117,9 @@ class TotalLinesMetric(Metric): description = "Total lines of code" unit = "lines" - def calculate(self, context: Context, metrics: dict) -> None: + def calculate(self, context: Context, measurements: dict) -> Measurement: # Count lines in all files - self.value = 1000 # simplified + return self.measurement(value=1000) # simplified class AverageLinesPerFileMetric(Metric): @@ -130,15 +131,14 @@ class AverageLinesPerFileMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [FileCountMetric, TotalLinesMetric] - def calculate(self, context: Context, metrics: dict) -> None: - file_count = metrics[FileCountMetric].value - total_lines = metrics[TotalLinesMetric].value + def calculate(self, context: Context, measurements: dict) -> Measurement: + file_count = measurements[FileCountMetric].value + total_lines = measurements[TotalLinesMetric].value if file_count > 0: - self.value = total_lines / file_count + return self.measurement(value=total_lines / file_count) else: - self.value = 0 - self.diagnostic = "No files found" + return self.measurement(value=0, diagnostic="No files found") ``` CheckUp automatically resolves dependencies and calculates metrics in the correct order. @@ -201,7 +201,7 @@ from pathlib import Path result = ( CheckHub(config_path=Path("checkup.yaml")) - .with_metrics([FileCountMetric, TotalLinesMetric]) + .with_metrics([FileCountMetric(), TotalLinesMetric()]) .measure() ) ``` diff --git a/docs/index.md b/docs/index.md index 87c46e5..33732c1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,7 +16,7 @@ CheckUp is an extensible Python framework designed to calculate metrics from con ## Quick Example ```python -from checkup import CheckHub, ConsoleMaterializer, Metric +from checkup import CheckHub, ConsoleMaterializer, Metric, Measurement from checkup.types import Context @@ -25,15 +25,14 @@ class SimpleMetric(Metric): description = "A simple example metric" unit = "count" - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 42 - self.diagnostic = "Calculated successfully" + def calculate(self, context: Context, measurements: dict) -> Measurement: + return self.measurement(value=42, diagnostic="Calculated successfully") # Run the metric and output to console ( CheckHub() - .with_metrics([SimpleMetric]) + .with_metrics([SimpleMetric()]) .measure() .materialize(ConsoleMaterializer(group_tag_1="domain", group_tag_2="project")) ) diff --git a/docs/plugins/conveyor.md b/docs/plugins/conveyor.md index 85724f1..52a2419 100644 --- a/docs/plugins/conveyor.md +++ b/docs/plugins/conveyor.md @@ -197,11 +197,13 @@ Check for errors in your metrics: ```python class SafeMetric(Metric): - def calculate(self, context, metrics): + def calculate(self, context, measurements): conveyor = context.get("conveyor", {}) if conveyor.get("error"): - self.value = None - self.diagnostic = f"API Error: {conveyor['error']}" - return + return self.measurement( + value=None, + diagnostic=f"API Error: {conveyor['error']}" + ) # Normal calculation... + return self.measurement(value=computed_value) ``` diff --git a/docs/plugins/overview.md b/docs/plugins/overview.md index 5c502f6..6a7b04f 100644 --- a/docs/plugins/overview.md +++ b/docs/plugins/overview.md @@ -37,7 +37,7 @@ from checkup_dbt import DbtProvider, ModelTestCoverageMetric # Use in your pipeline result = ( CheckHub() - .with_metrics([BranchCountMetric, CommitFrequencyMetric]) + .with_metrics([BranchCountMetric(), CommitFrequencyMetric()]) .with_providers([ [GitProvider(repo_path="/path/to/repo")] ]) @@ -114,7 +114,7 @@ class MyServiceProvider(Provider): ```python # src/checkup_myservice/metrics.py -from checkup import Metric, Provider +from checkup import Metric, Measurement, Provider from checkup.types import Context from .providers import MyServiceProvider @@ -128,9 +128,9 @@ class MyServiceMetric(Metric): def providers(cls) -> list[type[Provider]]: return [MyServiceProvider] - def calculate(self, context: Context, metrics: dict) -> None: + def calculate(self, context: Context, measurements: dict) -> Measurement: data = context["myservice"]["data"] - self.value = data.get("status", "unknown") + return self.measurement(value=data.get("status", "unknown")) ``` 5. Export in `__init__.py`: diff --git a/plugins/checkup-conveyor/README.md b/plugins/checkup-conveyor/README.md index e20a619..bd0d58b 100644 --- a/plugins/checkup-conveyor/README.md +++ b/plugins/checkup-conveyor/README.md @@ -23,13 +23,13 @@ from checkup_conveyor import ConveyorProvider results = ( CheckHub() .with_metrics([]) - .with_providers([ + .with_providers([[ ConveyorProvider( project_name="my-project", api_key="your-api-key", environment_name="production", ), - ]) + ]]) .measure() ) ``` @@ -51,11 +51,12 @@ class MyConveyorMetric(ConveyorMetric): name = "my_conveyor_metric" description = "My custom Conveyor metric" - def calculate(self, context, metrics): + def calculate(self, context, measurements): api_client = self.get_api_client(context) project_name = self.get_project_name(context) environment_name = self.get_environment_name(context) # Use api_client to fetch data from Conveyor - self.value = ... + value = ... + return self.measurement(value=value) ``` diff --git a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py index 5e65190..0eb96b0 100644 --- a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py +++ b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py @@ -4,6 +4,7 @@ import requests from checkup import Context +from checkup.metric import Measurement, Metric from checkup_conveyor import ConveyorMetric logger = logging.getLogger(__name__) @@ -13,13 +14,13 @@ class ConveyorLastDeploymentTime(ConveyorMetric): name: ClassVar[str] = "Conveyor Last Deployment Time" description: ClassVar[str] = "Time of the last deployment in Conveyor" unit: ClassVar[str] = "timestamp" - diagnostic: str = "Deploy the project again to update this value." - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: - self.value = None - return + return self.measurement(value=None) r = requests.get( f"{self.base_url}/projects/{proj_id}/deployments", headers=self.get_conveyor_api_headers(context), @@ -27,22 +28,24 @@ def calculate(self, context: Context, metrics: dict) -> None: deployments = r.get("deployment", []) if not deployments: logger.warning("No deployments found for project %s", proj_id) - self.value = None - return - self.value = deployments[0]["deployedOn"] + return self.measurement(value=None) + return self.measurement( + value=deployments[0]["deployedOn"], + diagnostic="Deploy the project again to update this value.", + ) class ConveyorIsDirtyDeployment(ConveyorMetric): name: ClassVar[str] = "Conveyor Is Dirty Deployment" description: ClassVar[str] = "True if the last deployment was dirty" unit: ClassVar[str] = "boolean" - diagnostic: str = "Commit changes to git, and deploy the project again." - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: - self.value = None - return + return self.measurement(value=None) r = requests.get( f"{self.base_url}/projects/{proj_id}/builds", headers=self.get_conveyor_api_headers(context), @@ -50,9 +53,12 @@ def calculate(self, context: Context, metrics: dict) -> None: builds = r.get("builds", []) if not builds: logger.warning("No builds found for project %s", proj_id) - self.value = None - return - self.value = builds[0]["gitHash"].endswith(".dirty") + return self.measurement(value=None) + is_dirty = builds[0]["gitHash"].endswith(".dirty") + diagnostic = ( + "Commit changes to git, and deploy the project again." if is_dirty else "" + ) + return self.measurement(value=is_dirty, diagnostic=diagnostic) class ConveyorLastRunStatus(ConveyorMetric): @@ -60,15 +66,15 @@ class ConveyorLastRunStatus(ConveyorMetric): description: ClassVar[str] = "Status of the last run in Conveyor" unit: ClassVar[str] = "string" - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: - self.value = None - return + return self.measurement(value=None) env_id = self.get_environment_id(context) if env_id is None: - self.value = None - return + return self.measurement(value=None) r = requests.get( f"{self.base_url}/environments/{env_id}/application_runs", params={ @@ -85,6 +91,5 @@ def calculate(self, context: Context, metrics: dict) -> None: proj_id, env_id, ) - self.value = None - return - self.value = runs[0]["phase"] + return self.measurement(value=None) + return self.measurement(value=runs[0]["phase"]) diff --git a/plugins/checkup-dbt/README.md b/plugins/checkup-dbt/README.md index efb40b8..68f4653 100644 --- a/plugins/checkup-dbt/README.md +++ b/plugins/checkup-dbt/README.md @@ -28,13 +28,13 @@ from checkup_dbt import ( results = ( CheckHub() .with_metrics([ - DbtModelsMetric, - DbtColumnsMetric, - DbtTestsMetric, + DbtModelsMetric(), + DbtColumnsMetric(), + DbtTestsMetric(), ]) - .with_providers([ + .with_providers([[ DbtManifestProvider(dbt_project_dir="./my_dbt_project"), - ]) + ]]) .measure() ) ``` @@ -170,7 +170,7 @@ class MyCustomDbtMetric(DbtMetric): name = "my_custom_metric" description = "My custom dbt metric" - def calculate(self, context, metrics): - manifest = context["dbt"]["manifest"] - self.value = len(manifest.nodes) + def calculate(self, context, measurements): + manifest = self.get_manifest(context) + return self.measurement(value=len(manifest.nodes)) ``` diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py index 2299af4..2a50793 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py @@ -9,7 +9,7 @@ from dbt.artifacts.resources.types import NodeType from dbt.contracts.graph.manifest import Manifest -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.provider import Provider from checkup.types import Context from checkup_dbt.manifest_query import ManifestQuery @@ -69,7 +69,8 @@ def query(self, context: Context) -> ManifestQuery: class DbtCountMetric(DbtMetric): - """Base class for metrics that count nodes or columns. + """ + Base class for metrics that count nodes or columns. Subclasses should define: - name, description, unit (ClassVars) @@ -86,22 +87,26 @@ class DbtCountMetric(DbtMetric): predicate: ClassVar[Callable[..., bool] | None] = None log_message: ClassVar[str] = "Found {value} items" - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: cls = type(self) query = self.query(context).filter_by_type(cls.resource_type) if cls.count_target == CountTarget.COLUMNS: - self.value = query.count_columns(cls.predicate) + value = query.count_columns(cls.predicate) else: if cls.predicate: query = query.filter(cls.predicate) - self.value = query.count() + value = query.count() - logger.info(cls.log_message.format(value=self.value)) + logger.info(cls.log_message.format(value=value)) + return self.measurement(value=value) class DbtDiagnosticMetric(DbtMetric): - """Base class for metrics that count and list items with diagnostics. + """ + Base class for metrics that count and list items with diagnostics. Produces both a count and a diagnostic listing the items. @@ -123,7 +128,9 @@ class DbtDiagnosticMetric(DbtMetric): log_message: ClassVar[str] = "Found {value} items" max_diagnostic_items: ClassVar[int] = 50 - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: cls = type(self) query = self.query(context).filter_by_type(cls.resource_type) @@ -134,13 +141,15 @@ def calculate(self, context: Context, metrics: dict) -> None: query = query.filter(cls.predicate) names = query.names() - self.value = len(names) + value = len(names) + diagnostic = "" if names: if (overflow := len(names) - cls.max_diagnostic_items) > 0: shown = ", ".join(names[: cls.max_diagnostic_items]) - self.diagnostic = ( + diagnostic = ( f"{cls.diagnostic_prefix}: {shown}, ... and {overflow} more" ) else: - self.diagnostic = f"{cls.diagnostic_prefix}: {', '.join(names)}" - logger.info(cls.log_message.format(value=self.value)) + diagnostic = f"{cls.diagnostic_prefix}: {', '.join(names)}" + logger.info(cls.log_message.format(value=value)) + return self.measurement(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py index 0295e64..79532a0 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py @@ -3,7 +3,7 @@ import yaml -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -11,7 +11,8 @@ class DbtFlaggedPackagesMetric(DbtMetric): - """Metric that counts flagged packages in packages.yml. + """ + Metric that counts flagged packages in packages.yml. Checks packages.yml for packages matching the configured flagged_packages list. Useful for identifying deprecated, insecure, or non-approved packages. @@ -30,23 +31,23 @@ class DeprecatedPackagesMetric(DbtFlaggedPackagesMetric): flagged_packages: list[str] - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: project_dir = self.get_project_dir(context) packages_path = project_dir / "packages.yml" if not packages_path.exists(): logger.warning(f"packages.yml not found at {packages_path}") - self.value = 0 - self.diagnostic = "packages.yml not found" - return + return self.measurement(value=0, diagnostic="packages.yml not found") with open(packages_path) as f: packages_data = yaml.safe_load(f) if not packages_data or "packages" not in packages_data: - self.value = 0 - self.diagnostic = "No packages defined in packages.yml" - return + return self.measurement( + value=0, diagnostic="No packages defined in packages.yml" + ) flagged = [] for package in packages_data["packages"]: @@ -56,7 +57,7 @@ def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> No flagged.append(package["git"]) break - self.value = len(flagged) - if flagged: - self.diagnostic = f"Flagged packages: {', '.join(flagged)}" - logger.info(f"Found {self.value} flagged packages") + value = len(flagged) + diagnostic = f"Flagged packages: {', '.join(flagged)}" if flagged else "" + logger.info(f"Found {value} flagged packages") + return self.measurement(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py index 85f3366..4a4402c 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py @@ -3,6 +3,7 @@ from dbt.artifacts.resources.types import NodeType +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric, NamingConventionChecker @@ -10,7 +11,8 @@ class DbtModelsNotAdheringToNamingConventionMetric(DbtMetric): - """Metric for checking model naming conventions. + """ + Metric for checking model naming conventions. This metric requires a custom checker function that defines the naming convention. Use with_checker() to create a configured metric class. @@ -37,7 +39,9 @@ def get_checker(cls) -> NamingConventionChecker: return CustomNamingConventionMetric - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: manifest = self.get_manifest(context) checker = self.get_checker() @@ -47,7 +51,9 @@ def calculate(self, context: Context, metrics: dict) -> None: if node.resource_type == NodeType.Model and not checker(context, node) ] - self.value = len(non_adhering_models) + value = len(non_adhering_models) + diagnostic = "" if non_adhering_models: - self.diagnostic = f"Models not adhering to naming convention: {', '.join(sorted(non_adhering_models))}" - logger.info(f"Found {self.value} models not adhering to naming convention") + diagnostic = f"Models not adhering to naming convention: {', '.join(sorted(non_adhering_models))}" + logger.info(f"Found {value} models not adhering to naming convention") + return self.measurement(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py index a9eb5e9..9a5976e 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py @@ -3,7 +3,7 @@ import yaml -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -11,7 +11,8 @@ class DbtProfileHostMetric(DbtMetric): - """Extracts the host value from profiles.yml. + """ + Extracts the host value from profiles.yml. Searches for a host configuration in the specified profile and target. If profile is not specified, searches all profiles. @@ -33,30 +34,28 @@ class ProdHostMetric(DbtProfileHostMetric): profile: str | None = None target: str - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: project_dir = self.get_project_dir(context) profiles_path = project_dir / "profiles.yml" if not profiles_path.exists(): logger.warning(f"profiles.yml not found at {profiles_path}") - self.value = None - self.diagnostic = "profiles.yml not found" - return + return self.measurement(value=None, diagnostic="profiles.yml not found") with open(profiles_path) as f: profiles = yaml.safe_load(f) if not profiles: - self.value = None - self.diagnostic = "profiles.yml is empty" - return + return self.measurement(value=None, diagnostic="profiles.yml is empty") host = self._find_host(profiles) - self.value = host if host: - self.diagnostic = f"Host: {host}" + diagnostic = f"Host: {host}" else: - self.diagnostic = f"No host found for target '{self.target}'" + diagnostic = f"No host found for target '{self.target}'" + return self.measurement(value=host, diagnostic=diagnostic) def _find_host(self, profiles: dict) -> str | None: """Find host in profiles matching the configuration.""" diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py index 5de4b04..0f1f5a5 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py @@ -1,7 +1,7 @@ import logging from typing import ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric from checkup_dbt.metrics.quality.version import DbtVersionMetric @@ -10,7 +10,9 @@ class DbtSupportedVersionMetric(DbtMetric): - """Metric for checking dbt version compatibility.""" + """ + Metric for checking dbt version compatibility. + """ name: ClassVar[str] = "dbt_supported_version" description: ClassVar[str] = "Whether dbt version meets minimum requirement" @@ -22,8 +24,10 @@ class DbtSupportedVersionMetric(DbtMetric): def depends_on(cls) -> list[type[Metric]]: return [DbtVersionMetric] - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: - version: str = metrics[DbtVersionMetric].value + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + version: str = measurements[DbtVersionMetric].value major_version = int(version.split(".")[0]) minor_version = int(version.split(".")[1]) @@ -32,10 +36,12 @@ def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> No supported = major_version == min_major and minor_version >= min_minor - self.value = 1 if supported else 0 + value = 1 if supported else 0 + diagnostic = "" if not supported: - self.diagnostic = ( + diagnostic = ( f"dbt version {version} does not meet minimum requirement of {self.min_version}. " f"Please upgrade dbt to version {self.min_version} or later." ) - logger.info(f"dbt version {version} supported: {bool(self.value)}") + logger.info(f"dbt version {version} supported: {bool(value)}") + return self.measurement(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py index 558664a..4b16458 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py @@ -3,7 +3,7 @@ import logging from typing import ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -11,14 +11,17 @@ class DbtVersionMetric(DbtMetric): - """Metric to detect the dbt version used to generate the manifest.""" + """ + Metric to detect the dbt version used to generate the manifest. + """ name: ClassVar[str] = "dbt_version" description: ClassVar[str] = "The dbt version used to generate the manifest" unit: ClassVar[str] = "version" - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: manifest = self.get_manifest(context) - self.value = manifest.metadata.dbt_version - self.diagnostic = f"dbt version: {self.value}" - logger.info(f"dbt version: {self.value}") + value = manifest.metadata.dbt_version + return self.measurement(value=value, diagnostic=f"dbt version: {value}") diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py index 1004c56..905148f 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py @@ -1,7 +1,7 @@ import logging from typing import ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric from checkup_dbt.metrics.core.columns import DbtColumnsMetric @@ -11,7 +11,8 @@ class DbtColumnTestCoverageMetric(DbtMetric): - """Percentage of columns with at least one test. + """ + Percentage of columns with at least one test. This is a derived metric that depends on other metrics, so it implements calculate() directly. @@ -25,13 +26,16 @@ class DbtColumnTestCoverageMetric(DbtMetric): def depends_on(cls) -> list[type[Metric]]: return [DbtTestedColumnsMetric, DbtColumnsMetric] - def calculate(self, context: Context, metrics: dict) -> None: - tested = metrics[DbtTestedColumnsMetric].value - total = metrics[DbtColumnsMetric].value + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + tested = measurements[DbtTestedColumnsMetric].value + total = measurements[DbtColumnsMetric].value if total > 0: - self.value = int(tested / total * 100) + value = int(tested / total * 100) else: - self.value = 0 + value = 0 - logger.info(f"Column test coverage: {self.value}%") + logger.info(f"Column test coverage: {value}%") + return self.measurement(value=value) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py index 465740a..e221ac1 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py @@ -3,6 +3,7 @@ from dbt.artifacts.resources.types import NodeType +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -20,7 +21,9 @@ class DbtTestedColumnsMetric(DbtMetric): description: ClassVar[str] = "Number of columns with at least one test" unit: ClassVar[str] = "columns" - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: manifest = self.get_manifest(context) all_columns = { @@ -40,5 +43,6 @@ def calculate(self, context: Context, metrics: dict) -> None: and node.column_name is not None } - self.value = len(all_columns & tested_columns) - logger.info(f"Found {self.value} tested columns") + value = len(all_columns & tested_columns) + logger.info(f"Found {value} tested columns") + return self.measurement(value=value) diff --git a/plugins/checkup-git/README.md b/plugins/checkup-git/README.md index 0c207b8..754ac40 100644 --- a/plugins/checkup-git/README.md +++ b/plugins/checkup-git/README.md @@ -27,12 +27,12 @@ from checkup_git import ( results = ( CheckHub() .with_metrics([ - GitDaysSinceLastUpdateMetric, - GitTrackedFileCountMetric, + GitDaysSinceLastUpdateMetric(), + GitTrackedFileCountMetric(), ]) - .with_providers([ + .with_providers([[ GitProvider("./my_repo"), - ]) + ]]) .measure() ) ``` @@ -73,8 +73,9 @@ class MyCustomGitMetric(GitMetric): name = "my_custom_metric" description = "My custom git metric" - def calculate(self, context, metrics): + def calculate(self, context, measurements): git_context = self.get_context(context) tracked_files = git_context.get("git_tracked_files", []) - self.value = len([f for f in tracked_files if f.endswith(".py")]) + python_files = [f for f in tracked_files if f.endswith(".py")] + return self.measurement(value=len(python_files)) ``` diff --git a/plugins/checkup-git/src/checkup_git/metrics.py b/plugins/checkup-git/src/checkup_git/metrics.py index f8a0559..f8ac00c 100644 --- a/plugins/checkup-git/src/checkup_git/metrics.py +++ b/plugins/checkup-git/src/checkup_git/metrics.py @@ -4,7 +4,7 @@ from fnmatch import fnmatch from typing import ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.provider import Provider from checkup.types import Context from checkup_git.provider import GitProvider @@ -29,19 +29,21 @@ class GitDaysSinceLastUpdateMetric(GitMetric): description: ClassVar[str] = "Days since the last git commit" unit: ClassVar[str] = "days" - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: git_context = self.get_context(context) last_commit_date = git_context.get("git_last_commit_date") if not isinstance(last_commit_date, datetime): - self.value = None - self.diagnostic = "No commits found" - return + return self.measurement(value=None, diagnostic="No commits found") now = datetime.now(UTC) delta = now - last_commit_date - self.value = delta.days - self.diagnostic = f"Last commit: {last_commit_date.strftime('%Y-%m-%d')}" + return self.measurement( + value=delta.days, + diagnostic=f"Last commit: {last_commit_date.strftime('%Y-%m-%d')}", + ) class GitTrackedFileCountMetric(GitMetric): @@ -65,21 +67,26 @@ class PythonTestFileCountMetric(GitTrackedFileCountMetric): pattern: str = "*" - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: git_context = self.get_context(context) tracked_files = git_context.get("git_tracked_files", []) if not isinstance(tracked_files, list): - self.value = 0 - self.diagnostic = "No git repository found" - return + return self.measurement(value=0, diagnostic="No git repository found") if self.pattern != "*": matched_files = [f for f in tracked_files if fnmatch(f, self.pattern)] - self.value = len(matched_files) if matched_files: - self.diagnostic = f"Matched files: {', '.join(matched_files)}" + return self.measurement( + value=len(matched_files), + diagnostic=f"Matched files: {', '.join(matched_files)}", + ) else: - self.diagnostic = f"No files matching pattern: {self.pattern}" + return self.measurement( + value=len(matched_files), + diagnostic=f"No files matching pattern: {self.pattern}", + ) else: - self.value = len(tracked_files) + return self.measurement(value=len(tracked_files)) diff --git a/plugins/checkup-python/src/checkup_python/metrics/version.py b/plugins/checkup-python/src/checkup_python/metrics/version.py index 876d8ed..1173466 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_python.metrics.utils import parse_semantic_version @@ -22,7 +22,9 @@ class PythonVersionMetric(Metric): description: ClassVar[str] = "The Python version configured for the project" unit: ClassVar[str] = "version" - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: path = None if "path" in context: @@ -36,7 +38,7 @@ def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> No or self._get_runtime_version() ) - self.value = version + return self.measurement(value=version) def _read_python_version_file(self, path: Path) -> str | None: """ diff --git a/plugins/checkup-python/src/checkup_python/metrics/version_check.py b/plugins/checkup-python/src/checkup_python/metrics/version_check.py index e7bfc07..a946dd8 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version_check.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version_check.py @@ -1,6 +1,6 @@ from typing import ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.types import Context from checkup_python.metrics.utils import parse_semantic_version from checkup_python.metrics.version import PythonVersionMetric @@ -25,12 +25,14 @@ class PythonVersionCheckMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [PythonVersionMetric] - def calculate(self, context: Context, metrics: dict[type[Metric], Metric]) -> None: - version_metric = metrics[PythonVersionMetric] - actual_version = version_metric.value + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + actual_version = measurements[PythonVersionMetric].value actual = parse_semantic_version(actual_version) min_ver = parse_semantic_version(self.min_version) max_ver = parse_semantic_version(self.max_version) - self.value = min_ver <= actual <= max_ver + value = min_ver <= actual <= max_ver + return self.measurement(value=value) From f72a9a01f4b6f0c678467f71475d6b9f2a5dbc6a Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Wed, 8 Apr 2026 16:22:38 +0200 Subject: [PATCH 06/21] update --- docs/concepts/materializers.md | 6 +- src/checkup/__init__.py | 3 +- src/checkup/executor.py | 301 +++++++++++----------- src/checkup/hub.py | 84 +++--- src/checkup/materializers/__init__.py | 10 +- src/checkup/materializers/base.py | 80 +++--- src/checkup/materializers/csv_file.py | 26 +- src/checkup/materializers/database.py | 36 ++- src/checkup/materializers/html_report.py | 18 +- src/checkup/metric.py | 75 ++++-- src/checkup/templates/metrics_report.html | 44 ++-- tests/conftest.py | 7 +- tests/fixtures.py | 245 +++++++++++------- tests/test_executors.py | 171 +++++++----- tests/test_graph.py | 46 ++-- tests/test_hub.py | 83 +++--- tests/test_hub_execution.py | 124 +++++---- tests/test_hub_validation.py | 8 +- tests/test_integration.py | 26 +- tests/test_metric.py | 38 +-- 20 files changed, 793 insertions(+), 638 deletions(-) diff --git a/docs/concepts/materializers.md b/docs/concepts/materializers.md index 217de86..bfd72ee 100644 --- a/docs/concepts/materializers.md +++ b/docs/concepts/materializers.md @@ -180,10 +180,10 @@ class JSONMaterializer(Materializer): data = [ { - "name": m.metric_name, + "name": m.metric.name, "value": m.value, - "unit": m.metric_unit, - "description": m.metric_description, + "unit": m.metric.unit, + "description": m.metric.description, "diagnostic": m.diagnostic, "tags": m.tags, } diff --git a/src/checkup/__init__.py b/src/checkup/__init__.py index 58f932d..842a796 100644 --- a/src/checkup/__init__.py +++ b/src/checkup/__init__.py @@ -14,7 +14,7 @@ Materializer, SQLAlchemyMaterializer, ) -from checkup.metric import ExecutorType, Metric +from checkup.metric import ExecutorType, Measurement, Metric from checkup.provider import Provider from checkup.providers.tags import TagProvider from checkup.types import Context @@ -25,6 +25,7 @@ "CheckHub", "MeasurementResult", "Metric", + "Measurement", "ExecutorType", "Provider", "TagProvider", diff --git a/src/checkup/executor.py b/src/checkup/executor.py index 6d6e762..1fb4dfc 100644 --- a/src/checkup/executor.py +++ b/src/checkup/executor.py @@ -7,7 +7,7 @@ from typing import Any from checkup.errors import ProviderError -from checkup.metric import ExecutorType, Metric +from checkup.metric import ExecutorType, Measurement, Metric from checkup.provider import Provider from checkup.types import Context from checkup.validators import validate_pickleable @@ -16,35 +16,28 @@ def _calculate_metric_in_process( - metric_cls: type[Metric], - config: dict, + metric: Metric, context: Context, tags: dict[str, Any], - calculated_data: dict[type[Metric], dict], -) -> Metric: + calculated: dict[type[Metric], Measurement], +) -> Measurement: """Calculate a single metric in a subprocess. This is a module-level function for ProcessPoolExecutor compatibility. Args: - metric_cls: Metric class to instantiate and calculate - config: Config dict for the metric + metric: Metric instance to calculate context: Context dict from providers tags: Tags dict to merge into metrics - calculated_data: Dict mapping metric classes to their serialized data + calculated: Dict mapping metric classes to Measurements Returns: - Calculated metric instance + Calculated Measurement """ - # Reconstruct calculated metrics from serialized data - calculated: dict[type[Metric], Metric] = {} - for cls, data in calculated_data.items(): - calculated[cls] = cls(**data) - metric = metric_cls(**config) - metric.tags.update(tags) - metric.calculate(context, calculated) - return metric + measurement = metric.calculate(context, calculated) + measurement.tags.update(tags) + return measurement class ProviderExecutor: @@ -92,28 +85,22 @@ def execute( class MetricCalculator: """Calculates metrics for a given context.""" - def __init__(self, metric_configs: dict | None = None): - """Initialize calculator with optional metric configs. - - Args: - metric_configs: Optional dict mapping metric names to config dicts - """ - self._configs = metric_configs or {} - def calculate( self, + metrics: list[Metric], execution_order: list[type[Metric]], context: Context, tags: dict[str, Any], provided_classes: set[type[Provider]], failed_providers: dict[type[Provider], ProviderError] | None = None, - ) -> list[Metric]: + ) -> list[Measurement]: """Calculate all metrics in execution order. Metrics are batched by executor type for efficient execution. Each batch is executed using the appropriate executor. Args: + metrics: List of metric instances to calculate execution_order: Topologically sorted metric classes context: Context dict from providers tags: Tags dict to merge into metrics @@ -121,14 +108,22 @@ def calculate( failed_providers: Dict mapping failed provider classes to their errors Returns: - List of calculated metrics + List of Measurements """ failed_providers = failed_providers or {} logger.debug("Starting metric calculation for %d metrics", len(execution_order)) - calculated: dict[type[Metric], Metric] = {} + + class_to_instance: dict[type[Metric], Metric] = {type(m): m for m in metrics} + + for metric_cls in execution_order: + if metric_cls not in class_to_instance: + # Instantiate dependent metrics implicitly with default constructor. + class_to_instance[metric_cls] = metric_cls() + + calculated: dict[type[Metric], Measurement] = {} skipped: set[type[Metric]] = set() failed: set[type[Metric]] = set() - result_metrics: list[Metric] = [] + result_measurements: list[Measurement] = [] i = 0 batch_num = 0 @@ -141,26 +136,28 @@ def calculate( i += 1 continue + metric = class_to_instance[metric_cls] + # Check for failed provider dependencies failed_deps = self._get_failed_providers( metric_cls, failed_providers, failed ) if failed_deps: - metric = self._create_failed_metric(metric_cls, tags, failed_deps) - calculated[metric_cls] = metric - result_metrics.append(metric) + measurement = self._create_failed_measurement(metric, tags, failed_deps) + calculated[metric_cls] = measurement + result_measurements.append(measurement) failed.add(metric_cls) i += 1 continue # Start a new batch with this executor type current_executor = metric_cls.executor - batch: list[type[Metric]] = [metric_cls] + batch: list[Metric] = [metric] i += 1 # Add more metrics to batch if they have same executor type # and don't depend on any metrics in the current batch - batch_set = set(batch) + batch_classes = {metric_cls} while i < len(execution_order): next_cls = execution_order[i] @@ -169,13 +166,17 @@ def calculate( i += 1 continue + next_metric = class_to_instance[next_cls] + failed_deps = self._get_failed_providers( next_cls, failed_providers, failed ) if failed_deps: - metric = self._create_failed_metric(next_cls, tags, failed_deps) - calculated[next_cls] = metric - result_metrics.append(metric) + measurement = self._create_failed_measurement( + next_metric, tags, failed_deps + ) + calculated[next_cls] = measurement + result_measurements.append(measurement) failed.add(next_cls) i += 1 continue @@ -184,11 +185,11 @@ def calculate( break # Different executor, start new batch # Check if this metric depends on any metric in the current batch - if set(next_cls.depends_on()) & batch_set: + if set(next_cls.depends_on()) & batch_classes: break # Has dependency in batch, must process batch first - batch.append(next_cls) - batch_set.add(next_cls) + batch.append(next_metric) + batch_classes.add(next_cls) i += 1 # Execute the batch with appropriate executor @@ -202,22 +203,22 @@ def calculate( batch_results = self._execute_batch( batch, current_executor, context, tags, calculated ) - for metric_cls_result, metric in batch_results.items(): - calculated[metric_cls_result] = metric - result_metrics.append(metric) + for metric_cls_key, measurement in batch_results.items(): + calculated[metric_cls_key] = measurement + result_measurements.append(measurement) logger.debug( "Metric %s calculated: value=%s", - metric_cls_result.name, - metric.value, + metric_cls_key.name, + measurement.value, ) logger.info( "Metric calculation complete: %d calculated, %d skipped, %d failed", - len(result_metrics) - len(failed), + len(result_measurements) - len(failed), len(skipped), len(failed), ) - return result_metrics + return result_measurements def _should_skip( self, @@ -285,52 +286,54 @@ def _get_failed_providers( return failed_names - def _create_failed_metric( + def _create_failed_measurement( self, - metric_cls: type[Metric], + metric: Metric, tags: dict[str, Any], failed_deps: list[str], - ) -> Metric: - """Create a metric instance with null value due to failed dependencies. + ) -> Measurement: + """Create a Measurement with null value due to failed dependencies. Args: - metric_cls: Metric class to instantiate - tags: Tags to merge into the metric + metric: Metric instance + tags: Tags to merge into the measurement failed_deps: List of failed dependency names Returns: - Metric instance with value=None and diagnostic explaining failure + Measurement with value=None and diagnostic explaining failure """ - metric = metric_cls(**self._configs.get(metric_cls.name, {})) - metric.tags.update(tags) - metric.value = None - metric.diagnostic = f"Failed: {', '.join(failed_deps)} failed" + diagnostic = f"Failed: {', '.join(failed_deps)} failed" logger.debug( "Metric %s marked as failed: %s", - metric_cls.name, - metric.diagnostic, + metric.name, + diagnostic, + ) + return Measurement( + metric=metric, + value=None, + tags=dict(tags), + diagnostic=diagnostic, ) - return metric def _execute_batch( self, - batch: list[type[Metric]], + batch: list[Metric], executor_type: ExecutorType, context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], Metric], - ) -> dict[type[Metric], Metric]: + calculated: dict[type[Metric], Measurement], + ) -> dict[type[Metric], Measurement]: """Execute a batch of metrics with the appropriate executor. Args: - batch: List of metric classes to execute + batch: List of metric instances to execute executor_type: Type of executor to use context: Context dict from providers - tags: Tags dict to merge into metrics - calculated: Dict of already-calculated metrics + tags: Tags dict to merge into measurements + calculated: Dict of already-calculated measurements (keyed by class) Returns: - Dict mapping metric classes to calculated metric instances + Dict mapping metric classes to Measurements """ if executor_type == ExecutorType.THREAD: return self._execute_batch_thread(batch, context, tags, calculated) @@ -343,95 +346,93 @@ def _execute_batch( def _execute_batch_thread( self, - batch: list[type[Metric]], + batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], Metric], - ) -> dict[type[Metric], Metric]: + calculated: dict[type[Metric], Measurement], + ) -> dict[type[Metric], Measurement]: """Execute metrics using ThreadPoolExecutor. Args: - batch: List of metric classes to execute + batch: List of metric instances to execute context: Context dict from providers - tags: Tags dict to merge into metrics - calculated: Dict of already-calculated metrics + tags: Tags dict to merge into measurements + calculated: Dict of already-calculated measurements (keyed by class) Returns: - Dict mapping metric classes to calculated metric instances + Dict mapping metric classes to Measurements """ - results: dict[type[Metric], Metric] = {} + results: dict[type[Metric], Measurement] = {} with ThreadPoolExecutor(max_workers=len(batch)) as executor: - future_to_cls = { + future_to_metric = { executor.submit( self._calculate_single_metric, - metric_cls, + metric, context, tags, calculated, - ): metric_cls - for metric_cls in batch + ): metric + for metric in batch } - for future in as_completed(future_to_cls): - metric_cls = future_to_cls[future] - metric = future.result() - results[metric_cls] = metric - # Update calculated for subsequent metrics in the same batch - calculated[metric_cls] = metric + for future in as_completed(future_to_metric): + metric = future_to_metric[future] + measurement = future.result() + results[type(metric)] = measurement + calculated[type(metric)] = measurement return results def _execute_batch_process( self, - batch: list[type[Metric]], + batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], Metric], - ) -> dict[type[Metric], Metric]: + calculated: dict[type[Metric], Measurement], + ) -> dict[type[Metric], Measurement]: """Execute metrics using ProcessPoolExecutor. Args: - batch: List of metric classes to execute + batch: List of metric instances to execute context: Context dict from providers - tags: Tags dict to merge into metrics - calculated: Dict of already-calculated metrics + tags: Tags dict to merge into measurements + calculated: Dict of already-calculated measurements (keyed by class) Returns: - Dict mapping metric classes to calculated metric instances + Dict mapping metric classes to Measurements Raises: - MetricPicklingError: If a metric class cannot be pickled + MetricPicklingError: If a metric cannot be pickled """ - # Validate all metrics in batch are pickleable before starting - for metric_cls in batch: - validate_pickleable(metric_cls) - results: dict[type[Metric], Metric] = {} + for metric in batch: + validate_pickleable(type(metric)) + + results: dict[type[Metric], Measurement] = {} with ProcessPoolExecutor(max_workers=len(batch)) as executor: - future_to_cls = { + future_to_metric = { executor.submit( _calculate_metric_in_process, - metric_cls, - self._configs.get(metric_cls.name, {}), + metric, context, tags, - {cls: m.model_dump() for cls, m in calculated.items()}, - ): metric_cls - for metric_cls in batch + calculated, + ): metric + for metric in batch } - for future in as_completed(future_to_cls): - metric_cls = future_to_cls[future] + for future in as_completed(future_to_metric): + metric = future_to_metric[future] try: - metric = future.result() - results[metric_cls] = metric - calculated[metric_cls] = metric + measurement = future.result() + results[type(metric)] = measurement + calculated[type(metric)] = measurement except Exception as e: logger.error( "Metric %s failed in process executor: %s", - metric_cls.name, + metric.name, e, ) raise @@ -440,23 +441,23 @@ def _execute_batch_process( def _execute_batch_asyncio( self, - batch: list[type[Metric]], + batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], Metric], - ) -> dict[type[Metric], Metric]: + calculated: dict[type[Metric], Measurement], + ) -> dict[type[Metric], Measurement]: """Execute metrics using asyncio. Supports both sync and async calculate methods. Args: - batch: List of metric classes to execute + batch: List of metric instances to execute context: Context dict from providers - tags: Tags dict to merge into metrics - calculated: Dict of already-calculated metrics + tags: Tags dict to merge into measurements + calculated: Dict of already-calculated measurements (keyed by class) Returns: - Dict mapping metric classes to calculated metric instances + Dict mapping metric classes to Measurements """ return asyncio.run( self._execute_batch_asyncio_impl(batch, context, tags, calculated) @@ -464,72 +465,72 @@ def _execute_batch_asyncio( async def _execute_batch_asyncio_impl( self, - batch: list[type[Metric]], + batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], Metric], - ) -> dict[type[Metric], Metric]: + calculated: dict[type[Metric], Measurement], + ) -> dict[type[Metric], Measurement]: """Async implementation of batch execution. Args: - batch: List of metric classes to execute + batch: List of metric instances to execute context: Context dict from providers - tags: Tags dict to merge into metrics - calculated: Dict of already-calculated metrics + tags: Tags dict to merge into measurements + calculated: Dict of already-calculated measurements (keyed by class) Returns: - Dict mapping metric classes to calculated metric instances + Dict mapping metric classes to Measurements """ - tasks = [] - for metric_cls in batch: - metric = metric_cls(**self._configs.get(metric_cls.name, {})) - metric.tags.update(tags) - tasks.append(self._calculate_async_metric(metric, context, calculated)) - - metrics = await asyncio.gather(*tasks) - return {type(m): m for m in metrics} + tasks = [ + self._calculate_async_metric(metric, context, tags, calculated) + for metric in batch + ] + results = await asyncio.gather(*tasks) + return {type(m): r for m, r in zip(batch, results, strict=True)} async def _calculate_async_metric( self, metric: Metric, context: Context, - calculated: dict[type[Metric], Metric], - ) -> Metric: + tags: dict[str, Any], + calculated: dict[type[Metric], Measurement], + ) -> Measurement: """Calculate a single metric, handling both sync and async calculate methods. Args: metric: Metric instance to calculate context: Context dict from providers - calculated: Dict of already-calculated metrics + tags: Tags dict to merge into measurements + calculated: Dict of already-calculated measurements (keyed by class) Returns: - Calculated metric instance + Measurement """ if inspect.iscoroutinefunction(metric.calculate): - await metric.calculate(context, calculated) + measurement = await metric.calculate(context, calculated) else: - metric.calculate(context, calculated) - return metric + measurement = metric.calculate(context, calculated) + measurement.tags.update(tags) + return measurement def _calculate_single_metric( self, - metric_cls: type[Metric], + metric: Metric, context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], Metric], - ) -> Metric: + calculated: dict[type[Metric], Measurement], + ) -> Measurement: """Calculate a single metric. Args: - metric_cls: Metric class to instantiate and calculate + metric: Metric instance to calculate context: Context dict from providers - tags: Tags dict to merge into metrics - calculated: Dict of already-calculated metrics + tags: Tags dict to merge into measurements + calculated: Dict of already-calculated measurements (keyed by class) Returns: - Calculated metric instance + Measurement """ - metric = metric_cls(**self._configs.get(metric_cls.name, {})) - metric.tags.update(tags) - metric.calculate(context, calculated) - return metric + measurement = metric.calculate(context, calculated) + measurement.tags.update(tags) + return measurement diff --git a/src/checkup/hub.py b/src/checkup/hub.py index 49f8c5e..746063c 100644 --- a/src/checkup/hub.py +++ b/src/checkup/hub.py @@ -1,19 +1,17 @@ -"""CheckHub main orchestration.""" +"""CheckHub orchestration.""" import logging import os from collections.abc import Iterable from concurrent.futures import ProcessPoolExecutor, as_completed -from pathlib import Path from typing import TYPE_CHECKING from pydantic import BaseModel, Field -from checkup.config import load_config from checkup.errors import DuplicateMetricNameError, MetricPicklingError, ProviderError from checkup.executor import MetricCalculator, ProviderExecutor from checkup.graph import build_dependency_graph, topological_sort -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.provider import Provider from checkup.validators import validate_providers, validate_unique_metric_names @@ -33,12 +31,13 @@ class MeasurementResult(BaseModel): - """Result of measuring metrics. + """ + Result of measuring metrics. - Contains all calculated metrics and any errors from failed contexts. + Contains all calculated measurements and any errors from failed contexts. """ - metrics: list[Metric] + measurements: list[Measurement] direct_metric_names: set[str] = Field(default_factory=set) errors: list[tuple[list[Provider], Exception]] = Field(default_factory=list) @@ -50,32 +49,35 @@ def materialize(self, materializer: "Materializer") -> None: Args: materializer: Materializer instance for output """ - materializer.materialize(self.metrics, self.direct_metric_names) + materializer.materialize(self.measurements, self.direct_metric_names) def _measure_single_provider_set( provider_set: list[Provider], + metrics: list[Metric], execution_order: list[type[Metric]], - metric_configs: dict, -) -> list[Metric]: +) -> list[Measurement]: """Calculate all metrics for a single provider set. This is a module-level function for ProcessPoolExecutor compatibility. Args: provider_set: List of provider instances + metrics: List of metric instances to calculate execution_order: Topologically sorted metric classes - metric_configs: Config dict for metrics Returns: - List of calculated metrics with tags merged + List of Measurements with tags merged Raises: ProviderError: If a provider fails during execution """ + context, tags, errors = ProviderExecutor().execute(provider_set) failed_providers = {type(e.provider): e for e in errors} - return MetricCalculator(metric_configs).calculate( + + return MetricCalculator().calculate( + metrics, execution_order, context, tags, @@ -85,30 +87,26 @@ def _measure_single_provider_set( class CheckHub: - """Main entry point for metrics calculation. + """ + Main entry point for metrics calculation. Usage: CheckHub() - .with_metrics([MetricA, MetricB]) + .with_metrics([MetricA(), MetricB(threshold=10)]) .measure() .materialize(ConsoleMaterializer()) """ - def __init__(self, config_path: Path | None = None) -> None: - """Initialize CheckHub. - - Args: - config_path: Optional path to YAML config file - """ - self._metrics: list[type[Metric]] = [] + def __init__(self) -> None: + """Initialize CheckHub.""" + self._metrics: list[Metric] = [] self._provider_sets: list[list[Provider]] = [] - self._config_path = config_path - def with_metrics(self, metrics: Iterable[type[Metric]]) -> "CheckHub": - """Register metrics to calculate. + def with_metrics(self, metrics: Iterable[Metric]) -> "CheckHub": + """Register metric instances to calculate. Args: - metrics: Iterable of metric classes + metrics: Iterable of metric instances (configured via constructor) Returns: Self for chaining @@ -142,7 +140,7 @@ def measure( max_workers: Max parallel workers. None = use all CPUs. Returns: - MeasurementResult containing all calculated metrics and errors + MeasurementResult containing all calculated measurements and errors Raises: DuplicateMetricNameError: If multiple metrics have the same name @@ -153,27 +151,15 @@ def measure( len(self._provider_sets), ) - metric_configs: dict = {} - if self._config_path: - logger.debug("Loading config from %s", self._config_path) - metric_configs = load_config(self._config_path) - - # Build dependency graph and get execution order + metric_classes = [type(m) for m in self._metrics] logger.debug("Building dependency graph") - execution_order = topological_sort(build_dependency_graph(self._metrics)) - - # Validate unique metric names across all metrics (including dependencies) + execution_order = topological_sort(build_dependency_graph(metric_classes)) validate_unique_metric_names(list(execution_order)) - direct_metric_names = {m.name for m in self._metrics} - - # Use empty provider set if none specified and none required provider_sets = self._provider_sets if self._provider_sets else [[]] - - # Validate providers before running validate_providers(list(execution_order), provider_sets) - all_metrics: list[Metric] = [] + all_measurements: list[Measurement] = [] all_errors: list[tuple[list[Provider], Exception]] = [] workers = max_workers if max_workers is not None else os.cpu_count() @@ -183,8 +169,8 @@ def measure( executor.submit( _measure_single_provider_set, provider_set=ps, + metrics=self._metrics, execution_order=execution_order, - metric_configs=metric_configs, ): ps for ps in provider_sets } @@ -192,7 +178,7 @@ def measure( for future in as_completed(future_to_provider_set): ps = future_to_provider_set[future] try: - all_metrics.extend(future.result()) + all_measurements.extend(future.result()) except Exception as e: logger.error("Provider set failed: %s", e) logger.debug("Provider set failure details:", exc_info=True) @@ -212,18 +198,18 @@ def measure( ) failed_contexts_str = "\n ".join(str(ctx) for ctx in failed_contexts) logger.info( - "Measurement complete: %d metrics calculated, %d failed contexts:\n %s", - len(all_metrics), + "Measurement complete: %d measurements, %d failed contexts:\n %s", + len(all_measurements), len(all_errors), failed_contexts_str, ) else: logger.info( - "Measurement complete: %d metrics calculated", - len(all_metrics), + "Measurement complete: %d measurements", + len(all_measurements), ) return MeasurementResult( - metrics=all_metrics, + measurements=all_measurements, direct_metric_names=direct_metric_names, errors=all_errors, ) diff --git a/src/checkup/materializers/__init__.py b/src/checkup/materializers/__init__.py index b184688..258ef7c 100644 --- a/src/checkup/materializers/__init__.py +++ b/src/checkup/materializers/__init__.py @@ -1,9 +1,9 @@ -"""Materializers for outputting metrics.""" +"""Materializers for outputting measurements.""" from checkup.materializers.base import ( Materializer, - group_metrics_by_tags, - group_metrics_hierarchical, + group_measurements_by_tags, + group_measurements_hierarchical, ) from checkup.materializers.console import ConsoleMaterializer from checkup.materializers.csv_file import CSVMaterializer @@ -16,6 +16,6 @@ "HTMLMaterializer", "Materializer", "SQLAlchemyMaterializer", - "group_metrics_by_tags", - "group_metrics_hierarchical", + "group_measurements_by_tags", + "group_measurements_hierarchical", ] diff --git a/src/checkup/materializers/base.py b/src/checkup/materializers/base.py index 914f2c1..d3ebf48 100644 --- a/src/checkup/materializers/base.py +++ b/src/checkup/materializers/base.py @@ -1,104 +1,108 @@ -"""Base materializer class and metric grouping helpers.""" +"""Base materializer class and measurement grouping helpers.""" from abc import ABC, abstractmethod from collections import defaultdict from pydantic import BaseModel -from checkup.metric import Metric +from checkup.metric import Measurement -def group_metrics_by_tags( - metrics: list[Metric], +def group_measurements_by_tags( + measurements: list[Measurement], tag1: str, tag2: str, default_value: str = "Unknown", -) -> dict[tuple[str, str], list[Metric]]: - """Group metrics by two tag values. +) -> dict[tuple[str, str], list[Measurement]]: + """Group measurements by two tag values. Args: - metrics: List of metrics to group + measurements: List of measurements to group tag1: First tag name for grouping tag2: Second tag name for grouping default_value: Value to use when tag is missing Returns: - Dict mapping (tag1_value, tag2_value) tuples to metric lists + Dict mapping (tag1_value, tag2_value) tuples to measurement lists """ - groups: dict[tuple[str, str], list[Metric]] = {} - for metric in metrics: - tag1_value = metric.tags.get(tag1, default_value) - tag2_value = metric.tags.get(tag2, default_value) + groups: dict[tuple[str, str], list[Measurement]] = {} + for measurement in measurements: + tag1_value = measurement.tags.get(tag1, default_value) + tag2_value = measurement.tags.get(tag2, default_value) key = (tag1_value, tag2_value) if key not in groups: groups[key] = [] - groups[key].append(metric) + groups[key].append(measurement) return groups -def group_metrics_hierarchical( - metrics: list[Metric], +def group_measurements_hierarchical( + measurements: list[Measurement], tag1: str, tag2: str, default_value: str = "Ungrouped", -) -> dict[str, dict[str, list[Metric]]]: - """Group metrics hierarchically by two tag values. +) -> dict[str, dict[str, list[Measurement]]]: + """Group measurements hierarchically by two tag values. Args: - metrics: List of metrics to group + measurements: List of measurements to group tag1: First tag name for top-level grouping tag2: Second tag name for nested grouping default_value: Value to use when tag is missing Returns: - Nested dict: {tag1_value: {tag2_value: [metrics]}} + Nested dict: {tag1_value: {tag2_value: [measurements]}} """ - grouped: dict[str, dict[str, list[Metric]]] = defaultdict(lambda: defaultdict(list)) + grouped: dict[str, dict[str, list[Measurement]]] = defaultdict( + lambda: defaultdict(list) + ) - for metric in metrics: - group1_value = metric.tags.get(tag1, default_value) - group2_value = metric.tags.get(tag2, default_value) - grouped[group1_value][group2_value].append(metric) + for measurement in measurements: + group1_value = measurement.tags.get(tag1, default_value) + group2_value = measurement.tags.get(tag2, default_value) + grouped[group1_value][group2_value].append(measurement) return dict(grouped) class Materializer(ABC, BaseModel): - """Base class for metric materializers. + """Base class for measurement materializers. - Materializers format and output metrics to various formats. + Materializers format and output measurements to various formats. Attributes: - include_indirect: If True, include metrics that were auto-added as + include_indirect: If True, include measurements that were auto-added as dependencies. If False (default), only include directly requested metrics. """ include_indirect: bool = False - def _filter_metrics( - self, metrics: list[Metric], direct_metric_names: set[str] - ) -> list[Metric]: - """Filter metrics based on include_indirect setting. + def _filter_measurements( + self, measurements: list[Measurement], direct_metric_names: set[str] + ) -> list[Measurement]: + """Filter measurements based on include_indirect setting. Args: - metrics: List of all calculated metrics + measurements: List of all calculated measurements direct_metric_names: Set of names of directly requested metrics Returns: - Filtered list of metrics + Filtered list of measurements """ if self.include_indirect: - return metrics - return [m for m in metrics if m.name in direct_metric_names] + return measurements + return [m for m in measurements if m.metric.name in direct_metric_names] @abstractmethod - def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: - """Format and output metrics. + def materialize( + self, measurements: list[Measurement], direct_metric_names: set[str] + ) -> None: + """Format and output measurements. Args: - metrics: List of calculated metrics + measurements: List of calculated measurements direct_metric_names: Set of names of directly requested metrics """ pass diff --git a/src/checkup/materializers/csv_file.py b/src/checkup/materializers/csv_file.py index f5caf68..6aebd9d 100644 --- a/src/checkup/materializers/csv_file.py +++ b/src/checkup/materializers/csv_file.py @@ -4,32 +4,34 @@ from pathlib import Path from checkup.materializers.base import Materializer -from checkup.metric import Metric +from checkup.metric import Measurement class CSVMaterializer(Materializer): - """Output metrics to a CSV file. + """Output measurements to a CSV file. - Writes metrics data in CSV format for further analysis. + Writes measurement data in CSV format for further analysis. """ output_path: Path - def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: - """Write metrics to CSV file.""" - filtered = self._filter_metrics(metrics, direct_metric_names) + def materialize( + self, measurements: list[Measurement], direct_metric_names: set[str] + ) -> None: + """Write measurements to CSV file.""" + filtered = self._filter_measurements(measurements, direct_metric_names) with open(self.output_path, "w", newline="") as f: writer = csv.writer(f) writer.writerow(["name", "value", "unit", "diagnostic", "description"]) - for metric in filtered: + for m in filtered: writer.writerow( [ - metric.name, - metric.value, - metric.unit, - metric.diagnostic, - metric.description, + m.metric.name, + m.value, + m.metric.unit, + m.diagnostic, + m.metric.description, ] ) diff --git a/src/checkup/materializers/database.py b/src/checkup/materializers/database.py index ae50f01..89a8b73 100644 --- a/src/checkup/materializers/database.py +++ b/src/checkup/materializers/database.py @@ -19,13 +19,13 @@ ) from checkup.materializers.base import Materializer -from checkup.metric import Metric +from checkup.metric import Measurement class SQLAlchemyMaterializer(Materializer): - """Output metrics to a database via SQLAlchemy. + """Output measurements to a database via SQLAlchemy. - Writes metrics as rows to a database table. The table is created + Writes measurements as rows to a database table. The table is created automatically if it doesn't exist. Rows are appended on each materialization, with a ``measured_at`` timestamp to distinguish runs. @@ -53,9 +53,11 @@ class SQLAlchemyMaterializer(Materializer): expand_tags: bool = False batch_size: int = 1000 - def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: - """Write metrics to the database.""" - filtered = self._filter_metrics(metrics, direct_metric_names) + def materialize( + self, measurements: list[Measurement], direct_metric_names: set[str] + ) -> None: + """Write measurements to the database.""" + filtered = self._filter_measurements(measurements, direct_metric_names) if not filtered: return @@ -74,9 +76,7 @@ def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> N ] if self.expand_tags: - tag_keys = { - key for metric in filtered if metric.tags for key in metric.tags - } + tag_keys = {key for m in filtered if m.tags for key in m.tags} tag_columns = [ Column(f"tag_{key}", String(255)) for key in sorted(tag_keys) ] @@ -96,22 +96,20 @@ def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> N now = datetime.now(UTC) rows = [] - for metric in filtered: + for m in filtered: row = { - "name": metric.name, - "value": str(metric.value) if metric.value is not None else None, - "unit": metric.unit, - "diagnostic": metric.diagnostic, - "description": metric.description, + "name": m.metric.name, + "value": str(m.value) if m.value is not None else None, + "unit": m.metric.unit, + "diagnostic": m.diagnostic, + "description": m.metric.description, "measured_at": now, } if self.expand_tags: for key in tag_keys: - row[f"tag_{key}"] = metric.tags.get(key) if metric.tags else None + row[f"tag_{key}"] = m.tags.get(key) if m.tags else None else: - row["tags"] = ( - json.dumps(metric.tags) if metric.tags is not None else None - ) + row["tags"] = json.dumps(m.tags) if m.tags is not None else None rows.append(row) with engine.connect() as conn: diff --git a/src/checkup/materializers/html_report.py b/src/checkup/materializers/html_report.py index 13ab6c7..a46f5c7 100644 --- a/src/checkup/materializers/html_report.py +++ b/src/checkup/materializers/html_report.py @@ -4,14 +4,14 @@ from jinja2 import Environment, FileSystemLoader -from checkup.materializers.base import Materializer, group_metrics_hierarchical -from checkup.metric import Metric +from checkup.materializers.base import Materializer, group_measurements_hierarchical +from checkup.metric import Measurement class HTMLMaterializer(Materializer): - """Output metrics to an HTML file with hierarchical grouping. + """Output measurements to an HTML file with hierarchical grouping. - Generates a styled HTML report with metrics grouped by two levels of tags. + Generates a styled HTML report with measurements grouped by two levels of tags. Uses Bootstrap accordions for collapsible groups. Attributes: @@ -24,12 +24,14 @@ class HTMLMaterializer(Materializer): group_tag_1: str group_tag_2: str - def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: + def materialize( + self, measurements: list[Measurement], direct_metric_names: set[str] + ) -> None: """Generate and write HTML report to file.""" - filtered = self._filter_metrics(metrics, direct_metric_names) + filtered = self._filter_measurements(measurements, direct_metric_names) - # Group metrics hierarchically - grouped = group_metrics_hierarchical( + # Group measurements hierarchically + grouped = group_measurements_hierarchical( filtered, self.group_tag_1, self.group_tag_2 ) diff --git a/src/checkup/metric.py b/src/checkup/metric.py index 3f5d9d6..987a6ec 100644 --- a/src/checkup/metric.py +++ b/src/checkup/metric.py @@ -1,4 +1,4 @@ -"""Metric base class.""" +"""Metric and Measurement classes.""" from abc import ABC, abstractmethod from enum import Enum @@ -13,7 +13,8 @@ class ExecutorType(Enum): - """Executor types for metric calculation. + """ + Executor types for metric calculation. Metrics can specify which executor to use for their calculation: - THREAD: ThreadPoolExecutor (default) - best for I/O-bound operations @@ -27,10 +28,11 @@ class ExecutorType(Enum): class Metric(ABC, BaseModel): - """Base class for all metrics. + """ + Base class for all metrics. - Metrics are Pydantic models that calculate values from context. - They can depend on other metrics and declare providers for context enrichment. + Metrics are immutable Pydantic models that define how to calculate values. + The calculate() method returns a Measurement with the result. """ name: ClassVar[str] @@ -38,26 +40,55 @@ class Metric(ABC, BaseModel): unit: ClassVar[str] executor: ClassVar[ExecutorType] = ExecutorType.THREAD - tags: dict = Field(default_factory=dict) - - value: Any = None - diagnostic: str = "" + model_config = {"frozen": True} # Make instances immutable @abstractmethod def calculate( - self, context: Context, metrics: dict[type["Metric"], "Metric"] - ) -> None: - """Calculate metric value and set self.value and self.diagnostic. + self, context: Context, measurements: dict[type["Metric"], "Measurement"] + ) -> "Measurement": + """ + Calculate metric and return a Measurement. Args: context: General context enriched by providers - metrics: Dict of already-calculated metric instances (dependencies) + measurements: Dict mapping Metric classes to their Measurements (dependencies) + + Returns: + Measurement with the calculated value """ pass + def measurement( + self, + value: Any = None, + tags: dict | None = None, + diagnostic: str = "", + ) -> "Measurement": + """ + Create a Measurement for this metric. + + Helper method to create a Measurement with this metric's metadata. + + Args: + value: The calculated value + tags: Optional tags dict (will be merged with provider tags) + diagnostic: Optional diagnostic message + + Returns: + Measurement instance + """ + + return Measurement( + metric=self, + value=value, + tags=tags or {}, + diagnostic=diagnostic, + ) + @classmethod def depends_on(cls) -> list[type["Metric"]]: - """Return list of metric classes this metric depends on. + """ + Return list of metric classes this metric depends on. Returns: List of metric classes (empty by default) @@ -66,9 +97,23 @@ def depends_on(cls) -> list[type["Metric"]]: @classmethod def providers(cls) -> list[type["Provider"]]: - """Return list of provider classes to enrich context. + """ + Return list of provider classes to enrich context. Returns: List of provider classes (empty by default) """ return [] + + +class Measurement(BaseModel): + """ + Result of a metric calculation. + + Holds the metric that produced it, the calculated value, tags, and diagnostic information. + """ + + metric: Metric + value: Any = None + tags: dict = Field(default_factory=dict) + diagnostic: str = "" diff --git a/src/checkup/templates/metrics_report.html b/src/checkup/templates/metrics_report.html index 0f6c61d..1fdaabf 100644 --- a/src/checkup/templates/metrics_report.html +++ b/src/checkup/templates/metrics_report.html @@ -38,13 +38,13 @@

Metrics Report

- +
{% for group1_name, group2_dict in grouped|dictsort %} {% set group1_idx = loop.index0 %}

-

-
- +
{% for group2_name, metrics in group2_dict|dictsort %}

-

-
- + @@ -83,39 +83,39 @@

- {% for metric in metrics|sort(attribute='name') %} - - - - + + + - + {% endfor %}
{{ metric.name }}{{ metric.value }}{{ metric.unit }}{{ measurement.metric.name }}{{ measurement.value }}{{ measurement.metric.unit }} - {{ metric.diagnostic }} - {% if metric.diagnostic %} + {{ measurement.diagnostic }} + {% if measurement.diagnostic %} {% endif %} {{ metric.description }}{{ measurement.metric.description }}
- +
{% endfor %}
- +
diff --git a/tests/conftest.py b/tests/conftest.py index 16e6639..44d5895 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,11 +9,10 @@ def dummy_metric(): @pytest.fixture -def dummy_metric_with_value(): - """Create a DummyMetric instance with value already calculated.""" +def dummy_measurement_with_value(): + """Create a Measurement from a DummyMetric with value already calculated.""" metric = DummyMetric(expected_value=10) - metric.calculate(context={}, metrics={}) - return metric + return metric.calculate(context={}, measurements={}) @pytest.fixture diff --git a/tests/fixtures.py b/tests/fixtures.py index fa3aa24..fe8257f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,6 +1,6 @@ from typing import Any, ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.provider import Provider from checkup.types import Context @@ -14,11 +14,13 @@ class DummyMetric(Metric): expected_value: int = 42 - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: """Set value to expected_value.""" - self.value = self.expected_value - self.diagnostic = ( - f"Dummy metric calculated with expected_value={self.expected_value}" + return self.measurement( + value=self.expected_value, + diagnostic=f"Dummy metric calculated with expected_value={self.expected_value}", ) @@ -34,11 +36,16 @@ def depends_on(cls) -> list[type[Metric]]: """Depends on DummyMetric.""" return [DummyMetric] - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: """Double the DummyMetric value.""" - base_value = metrics[DummyMetric].value - self.value = base_value * 2 - self.diagnostic = f"Doubled DummyMetric value from {base_value} to {self.value}" + base_value = measurements[DummyMetric].value + value = base_value * 2 + return self.measurement( + value=value, + diagnostic=f"Doubled DummyMetric value from {base_value} to {value}", + ) class Level2Metric(Metric): @@ -52,9 +59,13 @@ class Level2Metric(Metric): def depends_on(cls) -> list[type[Metric]]: return [DependentDummyMetric] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = metrics[DependentDummyMetric].value + 10 - self.diagnostic = f"Added 10 to DependentDummyMetric value: {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + value = measurements[DependentDummyMetric].value + 10 + return self.measurement( + value=value, diagnostic=f"Added 10 to DependentDummyMetric value: {value}" + ) class Level3Metric(Metric): @@ -68,10 +79,15 @@ class Level3Metric(Metric): def depends_on(cls) -> list[type[Metric]]: return [Level2Metric] - def calculate(self, context: Context, metrics: dict) -> None: - level2_value = metrics[Level2Metric].value - self.value = level2_value**2 - self.diagnostic = f"Squared Level2Metric value: {level2_value}^2 = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + level2_value = measurements[Level2Metric].value + value = level2_value**2 + return self.measurement( + value=value, + diagnostic=f"Squared Level2Metric value: {level2_value}^2 = {value}", + ) class CyclicMetricA(Metric): @@ -85,9 +101,10 @@ class CyclicMetricA(Metric): def depends_on(cls) -> list[type[Metric]]: return [CyclicMetricB] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 1 - self.diagnostic = "CyclicMetricA calculated" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement(value=1, diagnostic="CyclicMetricA calculated") class CyclicMetricB(Metric): @@ -101,9 +118,10 @@ class CyclicMetricB(Metric): def depends_on(cls) -> list[type[Metric]]: return [CyclicMetricA] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 1 - self.diagnostic = "CyclicMetricB calculated" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement(value=1, diagnostic="CyclicMetricB calculated") class RootA(Metric): @@ -114,9 +132,13 @@ class RootA(Metric): unit: ClassVar[str] = "count" base_value: int = 10 - def calculate(self, context: Context, metrics: dict) -> None: - self.value = self.base_value - self.diagnostic = f"RootA calculated with base_value={self.base_value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement( + value=self.base_value, + diagnostic=f"RootA calculated with base_value={self.base_value}", + ) class RootB(Metric): @@ -127,9 +149,13 @@ class RootB(Metric): unit: ClassVar[str] = "count" base_value: int = 20 - def calculate(self, context: Context, metrics: dict) -> None: - self.value = self.base_value - self.diagnostic = f"RootB calculated with base_value={self.base_value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement( + value=self.base_value, + diagnostic=f"RootB calculated with base_value={self.base_value}", + ) class RootC(Metric): @@ -140,9 +166,13 @@ class RootC(Metric): unit: ClassVar[str] = "count" base_value: int = 100 - def calculate(self, context: Context, metrics: dict) -> None: - self.value = self.base_value - self.diagnostic = f"RootC calculated with base_value={self.base_value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement( + value=self.base_value, + diagnostic=f"RootC calculated with base_value={self.base_value}", + ) class SharedAB(Metric): @@ -156,12 +186,15 @@ class SharedAB(Metric): def depends_on(cls) -> list[type[Metric]]: return [RootA, RootB] - def calculate(self, context: Context, metrics: dict) -> None: - root_a_val = metrics[RootA].value - root_b_val = metrics[RootB].value - self.value = root_a_val + root_b_val - self.diagnostic = ( - f"Sum of RootA ({root_a_val}) and RootB ({root_b_val}) = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + root_a_val = measurements[RootA].value + root_b_val = measurements[RootB].value + value = root_a_val + root_b_val + return self.measurement( + value=value, + diagnostic=f"Sum of RootA ({root_a_val}) and RootB ({root_b_val}) = {value}", ) @@ -176,10 +209,14 @@ class BranchB(Metric): def depends_on(cls) -> list[type[Metric]]: return [RootB] - def calculate(self, context: Context, metrics: dict) -> None: - root_b_val = metrics[RootB].value - self.value = root_b_val * 3 - self.diagnostic = f"Tripled RootB value: {root_b_val} * 3 = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + root_b_val = measurements[RootB].value + value = root_b_val * 3 + return self.measurement( + value=value, diagnostic=f"Tripled RootB value: {root_b_val} * 3 = {value}" + ) class LeafC(Metric): @@ -193,10 +230,14 @@ class LeafC(Metric): def depends_on(cls) -> list[type[Metric]]: return [RootC] - def calculate(self, context: Context, metrics: dict) -> None: - root_c_val = metrics[RootC].value - self.value = root_c_val**2 - self.diagnostic = f"Squared RootC value: {root_c_val}^2 = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + root_c_val = measurements[RootC].value + value = root_c_val**2 + return self.measurement( + value=value, diagnostic=f"Squared RootC value: {root_c_val}^2 = {value}" + ) class MidShared(Metric): @@ -210,11 +251,14 @@ class MidShared(Metric): def depends_on(cls) -> list[type[Metric]]: return [SharedAB] - def calculate(self, context: Context, metrics: dict) -> None: - shared_ab_val = metrics[SharedAB].value - self.value = shared_ab_val + 5 - self.diagnostic = ( - f"Added 5 to SharedAB value: {shared_ab_val} + 5 = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + shared_ab_val = measurements[SharedAB].value + value = shared_ab_val + 5 + return self.measurement( + value=value, + diagnostic=f"Added 5 to SharedAB value: {shared_ab_val} + 5 = {value}", ) @@ -229,10 +273,15 @@ class MidBranch(Metric): def depends_on(cls) -> list[type[Metric]]: return [BranchB] - def calculate(self, context: Context, metrics: dict) -> None: - branch_b_val = metrics[BranchB].value - self.value = branch_b_val * 2 - self.diagnostic = f"Doubled BranchB value: {branch_b_val} * 2 = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + branch_b_val = measurements[BranchB].value + value = branch_b_val * 2 + return self.measurement( + value=value, + diagnostic=f"Doubled BranchB value: {branch_b_val} * 2 = {value}", + ) class LeafAB(Metric): @@ -246,11 +295,16 @@ class LeafAB(Metric): def depends_on(cls) -> list[type[Metric]]: return [MidShared, MidBranch] - def calculate(self, context: Context, metrics: dict) -> None: - mid_shared_val = metrics[MidShared].value - mid_branch_val = metrics[MidBranch].value - self.value = mid_shared_val * mid_branch_val - self.diagnostic = f"Product of MidShared ({mid_shared_val}) and MidBranch ({mid_branch_val}) = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + mid_shared_val = measurements[MidShared].value + mid_branch_val = measurements[MidBranch].value + value = mid_shared_val * mid_branch_val + return self.measurement( + value=value, + diagnostic=f"Product of MidShared ({mid_shared_val}) and MidBranch ({mid_branch_val}) = {value}", + ) class DummyProvider(Provider): @@ -276,9 +330,13 @@ class ProviderDummyMetric(Metric): def providers(cls) -> list[type[Provider]]: return [DummyProvider] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = context[DummyProvider.name]["data"] - self.diagnostic = f"Retrieved dummy_data from context: {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + value = context[DummyProvider.name]["data"] + return self.measurement( + value=value, diagnostic=f"Retrieved dummy_data from context: {value}" + ) class FailingMetric(Metric): @@ -288,12 +346,12 @@ class FailingMetric(Metric): description: ClassVar[str] = "Fails when should_fail is True" unit: ClassVar[str] = "count" - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: if context.get("should_fail"): - self.diagnostic = "Metric failed as requested by context" raise ValueError("Intentional failure") - self.value = 1 - self.diagnostic = "Metric calculated successfully" + return self.measurement(value=1, diagnostic="Metric calculated successfully") class IntegrationProvider(Provider): @@ -320,10 +378,13 @@ class IntegrationBaseMetric(Metric): def providers(cls) -> list[type[Provider]]: return [IntegrationProvider] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = context[IntegrationProvider.name]["base_value"] - self.diagnostic = ( - f"Retrieved base_value from integration provider: {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + value = context[IntegrationProvider.name]["base_value"] + return self.measurement( + value=value, + diagnostic=f"Retrieved base_value from integration provider: {value}", ) @@ -339,10 +400,15 @@ class IntegrationDerivedMetric(Metric): def depends_on(cls): return [IntegrationBaseMetric] - def calculate(self, context: Context, metrics: dict) -> None: - base_val = metrics[IntegrationBaseMetric].value - self.value = base_val * self.multiplier - self.diagnostic = f"Multiplied base metric value: {base_val} * {self.multiplier} = {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + base_val = measurements[IntegrationBaseMetric].value + value = base_val * self.multiplier + return self.measurement( + value=value, + diagnostic=f"Multiplied base metric value: {base_val} * {self.multiplier} = {value}", + ) class PathLengthProvider(Provider): @@ -369,11 +435,14 @@ class PathMetric(Metric): def providers(cls) -> list[type[Provider]]: return [PathLengthProvider] - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: path_len = context[PathLengthProvider.name]["length"] - self.value = path_len * self.multiplier - self.diagnostic = ( - f"Path length {path_len} * multiplier {self.multiplier} = {self.value}" + value = path_len * self.multiplier + return self.measurement( + value=value, + diagnostic=f"Path length {path_len} * multiplier {self.multiplier} = {value}", ) @@ -386,11 +455,13 @@ class OtherDummyMetric(Metric): expected_value: int = 100 - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: """Set value to expected_value.""" - self.value = self.expected_value - self.diagnostic = ( - f"Other metric calculated with expected_value={self.expected_value}" + return self.measurement( + value=self.expected_value, + diagnostic=f"Other metric calculated with expected_value={self.expected_value}", ) @@ -403,9 +474,11 @@ class IndirectDummyMetric(Metric): expected_value: int = 100 - def calculate(self, context: Context, metrics: dict) -> None: + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: """Set value to expected_value.""" - self.value = self.expected_value - self.diagnostic = ( - f"Indirect metric calculated with expected_value={self.expected_value}" + return self.measurement( + value=self.expected_value, + diagnostic=f"Indirect metric calculated with expected_value={self.expected_value}", ) diff --git a/tests/test_executors.py b/tests/test_executors.py index 5376c83..46f7487 100644 --- a/tests/test_executors.py +++ b/tests/test_executors.py @@ -3,7 +3,7 @@ from typing import ClassVar from checkup.hub import CheckHub -from checkup.metric import ExecutorType, Metric +from checkup.metric import ExecutorType, Measurement, Metric from checkup.types import Context @@ -15,9 +15,10 @@ class ThreadMetric(Metric): unit: ClassVar[str] = "count" # executor defaults to ExecutorType.THREAD - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 10 - self.diagnostic = "Calculated in thread" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement(value=10, diagnostic="Calculated in thread") class ProcessMetric(Metric): @@ -28,9 +29,10 @@ class ProcessMetric(Metric): unit: ClassVar[str] = "count" executor: ClassVar[ExecutorType] = ExecutorType.PROCESS - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 20 - self.diagnostic = "Calculated in process" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement(value=20, diagnostic="Calculated in process") class AsyncMetric(Metric): @@ -41,9 +43,10 @@ class AsyncMetric(Metric): unit: ClassVar[str] = "count" executor: ClassVar[ExecutorType] = ExecutorType.ASYNCIO - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 30 - self.diagnostic = "Calculated with asyncio" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement(value=30, diagnostic="Calculated with asyncio") class AsyncMetricWithAsyncCalculate(Metric): @@ -54,12 +57,13 @@ class AsyncMetricWithAsyncCalculate(Metric): unit: ClassVar[str] = "count" executor: ClassVar[ExecutorType] = ExecutorType.ASYNCIO - async def calculate(self, context: Context, metrics: dict) -> None: + async def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: import asyncio await asyncio.sleep(0.001) # Small async operation - self.value = 40 - self.diagnostic = "Calculated with native async" + return self.measurement(value=40, diagnostic="Calculated with native async") class DependentThreadMetric(Metric): @@ -74,10 +78,14 @@ class DependentThreadMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [ThreadMetric] - def calculate(self, context: Context, metrics: dict) -> None: - base_value = metrics[ThreadMetric].value - self.value = base_value * 2 - self.diagnostic = f"Doubled thread metric: {base_value} -> {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + base_value = measurements[ThreadMetric].value + value = base_value * 2 + return self.measurement( + value=value, diagnostic=f"Doubled thread metric: {base_value} -> {value}" + ) class DependentProcessMetric(Metric): @@ -92,10 +100,14 @@ class DependentProcessMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [ThreadMetric] - def calculate(self, context: Context, metrics: dict) -> None: - base_value = metrics[ThreadMetric].value - self.value = base_value * 3 - self.diagnostic = f"Tripled thread metric: {base_value} -> {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + base_value = measurements[ThreadMetric].value + value = base_value * 3 + return self.measurement( + value=value, diagnostic=f"Tripled thread metric: {base_value} -> {value}" + ) class DependentAsyncMetric(Metric): @@ -110,10 +122,15 @@ class DependentAsyncMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [ProcessMetric] - def calculate(self, context: Context, metrics: dict) -> None: - base_value = metrics[ProcessMetric].value - self.value = base_value + 5 - self.diagnostic = f"Added 5 to process metric: {base_value} -> {self.value}" + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + base_value = measurements[ProcessMetric].value + value = base_value + 5 + return self.measurement( + value=value, + diagnostic=f"Added 5 to process metric: {base_value} -> {value}", + ) def test_default_executor_is_thread(): @@ -133,81 +150,89 @@ def test_asyncio_executor_specified(): def test_thread_metric_calculation(): """Test calculating a metric with ThreadPoolExecutor.""" - result = CheckHub().with_metrics([ThreadMetric]).measure() + result = CheckHub().with_metrics([ThreadMetric()]).measure() - assert len(result.metrics) == 1 - assert result.metrics[0].name == "thread_metric" - assert result.metrics[0].value == 10 + assert len(result.measurements) == 1 + assert result.measurements[0].metric.name == "thread_metric" + assert result.measurements[0].value == 10 def test_process_metric_calculation(): """Test calculating a metric with ProcessPoolExecutor.""" - result = CheckHub().with_metrics([ProcessMetric]).measure() + result = CheckHub().with_metrics([ProcessMetric()]).measure() - assert len(result.metrics) == 1 - assert result.metrics[0].name == "process_metric" - assert result.metrics[0].value == 20 + assert len(result.measurements) == 1 + assert result.measurements[0].metric.name == "process_metric" + assert result.measurements[0].value == 20 def test_asyncio_metric_calculation(): """Test calculating a metric with asyncio executor.""" - result = CheckHub().with_metrics([AsyncMetric]).measure() + result = CheckHub().with_metrics([AsyncMetric()]).measure() - assert len(result.metrics) == 1 - assert result.metrics[0].name == "async_metric" - assert result.metrics[0].value == 30 + assert len(result.measurements) == 1 + assert result.measurements[0].metric.name == "async_metric" + assert result.measurements[0].value == 30 def test_asyncio_metric_with_async_calculate(): """Test calculating a metric with native async calculate method.""" - result = CheckHub().with_metrics([AsyncMetricWithAsyncCalculate]).measure() + result = CheckHub().with_metrics([AsyncMetricWithAsyncCalculate()]).measure() - assert len(result.metrics) == 1 - assert result.metrics[0].name == "async_metric_native" - assert result.metrics[0].value == 40 + assert len(result.measurements) == 1 + assert result.measurements[0].metric.name == "async_metric_native" + assert result.measurements[0].value == 40 def test_mixed_executor_types(): """Test calculating metrics with different executor types.""" result = ( - CheckHub().with_metrics([ThreadMetric, ProcessMetric, AsyncMetric]).measure() + CheckHub() + .with_metrics([ThreadMetric(), ProcessMetric(), AsyncMetric()]) + .measure() ) - assert len(result.metrics) == 3 - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["thread_metric"].value == 10 - assert metrics_by_name["process_metric"].value == 20 - assert metrics_by_name["async_metric"].value == 30 + assert len(result.measurements) == 3 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["thread_metric"].value == 10 + assert measurements_by_name["process_metric"].value == 20 + assert measurements_by_name["async_metric"].value == 30 def test_thread_dependency_chain(): """Test dependency chain within thread executor.""" - result = CheckHub().with_metrics([DependentThreadMetric]).measure() + result = ( + CheckHub().with_metrics([DependentThreadMetric(), ThreadMetric()]).measure() + ) - assert len(result.metrics) == 2 - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["thread_metric"].value == 10 - assert metrics_by_name["dependent_thread"].value == 20 + assert len(result.measurements) == 2 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["thread_metric"].value == 10 + assert measurements_by_name["dependent_thread"].value == 20 def test_cross_executor_dependencies(): """Test dependencies across different executor types.""" - result = CheckHub().with_metrics([DependentProcessMetric]).measure() + result = ( + CheckHub().with_metrics([DependentProcessMetric(), ThreadMetric()]).measure() + ) - assert len(result.metrics) == 2 - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["thread_metric"].value == 10 - assert metrics_by_name["dependent_process"].value == 30 + assert len(result.measurements) == 2 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["thread_metric"].value == 10 + assert measurements_by_name["dependent_process"].value == 30 def test_async_depends_on_process(): """Test async metric depending on process metric.""" - result = CheckHub().with_metrics([DependentAsyncMetric]).measure() + result = ( + CheckHub().with_metrics([DependentAsyncMetric(), ProcessMetric()]).measure() + ) - assert len(result.metrics) == 2 - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["process_metric"].value == 20 - assert metrics_by_name["dependent_async"].value == 25 + assert len(result.measurements) == 2 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["process_metric"].value == 20 + assert measurements_by_name["dependent_async"].value == 25 def test_complex_mixed_dependencies(): @@ -215,7 +240,11 @@ def test_complex_mixed_dependencies(): result = ( CheckHub() .with_metrics( - [DependentThreadMetric, DependentProcessMetric, DependentAsyncMetric] + [ + DependentThreadMetric(), + DependentProcessMetric(), + DependentAsyncMetric(), + ] ) .measure() ) @@ -223,11 +252,11 @@ def test_complex_mixed_dependencies(): # ThreadMetric -> DependentThreadMetric (thread -> thread) # ThreadMetric -> DependentProcessMetric (thread -> process) # ProcessMetric -> DependentAsyncMetric (process -> async) - assert len(result.metrics) == 5 - - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["thread_metric"].value == 10 - assert metrics_by_name["dependent_thread"].value == 20 - assert metrics_by_name["dependent_process"].value == 30 - assert metrics_by_name["process_metric"].value == 20 - assert metrics_by_name["dependent_async"].value == 25 + assert len(result.measurements) == 5 + + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["thread_metric"].value == 10 + assert measurements_by_name["dependent_thread"].value == 20 + assert measurements_by_name["dependent_process"].value == 30 + assert measurements_by_name["process_metric"].value == 20 + assert measurements_by_name["dependent_async"].value == 25 diff --git a/tests/test_graph.py b/tests/test_graph.py index 967d810..986db28 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -22,7 +22,7 @@ ) from checkup.graph import build_dependency_graph, topological_sort -from checkup.metric import Metric +from checkup.metric import Measurement, Metric # ============================================================================= # build_dependency_graph tests @@ -100,18 +100,20 @@ def test_deep_chain_calculation(empty_context): DummyMetric(10) → DependentDummyMetric(20) → Level2Metric(30) → Level3Metric(900) """ + from checkup.metric import Measurement + graph = build_dependency_graph([Level3Metric]) order = topological_sort(graph) - calculated: dict[type[Metric], Metric] = {} + calculated: dict[type[Metric], Measurement] = {} for metric_cls in order: if metric_cls is DummyMetric: metric: Metric = DummyMetric(expected_value=10) else: metric = metric_cls() # type: ignore - metric.calculate(empty_context, calculated) - calculated[metric_cls] = metric + measurement = metric.calculate(empty_context, calculated) + calculated[metric_cls] = measurement assert calculated[DummyMetric].value == 10 assert calculated[DependentDummyMetric].value == 20 # 10 * 2 @@ -221,12 +223,12 @@ def test_complex_graph_calculation(empty_context): graph = build_dependency_graph([LeafAB, LeafC]) order = topological_sort(graph) - calculated: dict[type[Metric], Metric] = {} + calculated: dict[type[Metric], Measurement] = {} for metric_cls in order: metric = metric_cls() # type: ignore - metric.calculate(empty_context, calculated) - calculated[metric_cls] = metric + measurement = metric.calculate(empty_context, calculated) + calculated[metric_cls] = measurement # Verify all calculations assert calculated[RootA].value == 10 @@ -244,19 +246,19 @@ def test_complex_graph_via_checkhub(): """Test complex graph calculation through CheckHub.""" from checkup import CheckHub - result = CheckHub().with_metrics([LeafAB, LeafC]).measure() + result = CheckHub().with_metrics([LeafAB(), LeafC()]).measure() - metrics_by_name = {m.name: m for m in result.metrics} + measurements_by_name = {m.metric.name: m for m in result.measurements} # All 9 metrics returned (direct and indirect) - assert len(result.metrics) == 9 + assert len(result.measurements) == 9 # Only requested metrics are marked as direct assert result.direct_metric_names == {"leaf_ab", "leaf_c"} # Verify calculated values - assert metrics_by_name["leaf_ab"].value == 4200 - assert metrics_by_name["leaf_c"].value == 10000 + assert measurements_by_name["leaf_ab"].value == 4200 + assert measurements_by_name["leaf_c"].value == 10000 def test_shared_ancestor_calculated_once(empty_context): @@ -265,18 +267,20 @@ def test_shared_ancestor_calculated_once(empty_context): This verifies the framework doesn't re-calculate metrics that appear multiple times in the dependency graph. """ + from checkup.metric import Measurement + graph = build_dependency_graph([LeafAB]) order = topological_sort(graph) # Track how many times each metric class is calculated calculation_counts: dict[type[Metric], int] = {} - calculated: dict[type[Metric], Metric] = {} + calculated: dict[type[Metric], Measurement] = {} for metric_cls in order: calculation_counts[metric_cls] = calculation_counts.get(metric_cls, 0) + 1 metric = metric_cls() # type: ignore - metric.calculate(empty_context, calculated) - calculated[metric_cls] = metric + measurement = metric.calculate(empty_context, calculated) + calculated[metric_cls] = measurement # Each metric should appear exactly once in the execution order for metric_cls, count in calculation_counts.items(): @@ -291,15 +295,15 @@ def test_independent_subgraphs(): from checkup import CheckHub # Only request LeafC (independent subgraph) - returns LeafC and RootC - result = CheckHub().with_metrics([LeafC]).measure() + result = CheckHub().with_metrics([LeafC()]).measure() - assert len(result.metrics) == 2 + assert len(result.measurements) == 2 assert result.direct_metric_names == {"leaf_c"} - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["leaf_c"].value == 10000 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["leaf_c"].value == 10000 # Request both subgraphs - returns all 9 metrics - result = CheckHub().with_metrics([LeafAB, LeafC]).measure() + result = CheckHub().with_metrics([LeafAB(), LeafC()]).measure() - assert len(result.metrics) == 9 + assert len(result.measurements) == 9 assert result.direct_metric_names == {"leaf_ab", "leaf_c"} diff --git a/tests/test_hub.py b/tests/test_hub.py index 600e2b5..cd86501 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -2,7 +2,6 @@ import sys from io import StringIO -from pathlib import Path from fixtures import ( DependentDummyMetric, @@ -24,52 +23,54 @@ def test_checkhub_creation(): def test_checkhub_with_metrics(): """Test registering metrics with CheckHub.""" - hub = CheckHub().with_metrics([DummyMetric]) + hub = CheckHub().with_metrics([DummyMetric()]) assert isinstance(hub, CheckHub) def test_measurement_result_creation(): """Test creating a MeasurementResult.""" - metric = DummyMetric(expected_value=42) - result = MeasurementResult(metrics=[metric]) + metric = DummyMetric() + measurement = metric.measurement(value=42) + result = MeasurementResult(measurements=[measurement]) - assert len(result.metrics) == 1 - assert result.metrics[0] == metric + assert len(result.measurements) == 1 + assert result.measurements[0] == measurement def test_measurement_result_with_errors(): """Test MeasurementResult can hold errors.""" from checkup.providers.tags import TagProvider - metric = DummyMetric(expected_value=42) - metric.value = 42 + metric = DummyMetric() + measurement = metric.measurement(value=42) provider = TagProvider(path="/bad/path") errors = [([provider], ValueError("Path not found"))] - result = MeasurementResult(metrics=[metric], errors=errors) + result = MeasurementResult(measurements=[measurement], errors=errors) - assert len(result.metrics) == 1 + assert len(result.measurements) == 1 assert len(result.errors) == 1 assert result.errors[0][0] == [provider] def test_measurement_result_errors_default_empty(): """Test MeasurementResult.errors defaults to empty list.""" - metric = DummyMetric(expected_value=42) - result = MeasurementResult(metrics=[metric]) + metric = DummyMetric() + measurement = metric.measurement(value=42) + result = MeasurementResult(measurements=[measurement]) assert result.errors == [] def test_checkhub_measure_simple(): """Test measuring a single metric with no dependencies.""" - result = CheckHub().with_metrics([DummyMetric]).measure() + result = CheckHub().with_metrics([DummyMetric()]).measure() - assert len(result.metrics) == 1 - assert result.metrics[0].name == "dummy" - assert result.metrics[0].value == 42 + assert len(result.measurements) == 1 + assert result.measurements[0].metric.name == "dummy" + assert result.measurements[0].value == 42 def test_checkhub_measure_with_dependencies(): @@ -77,17 +78,17 @@ def test_checkhub_measure_with_dependencies(): DependentDummyMetric depends on DummyMetric and doubles its value. """ - result = CheckHub().with_metrics([DependentDummyMetric]).measure() + result = CheckHub().with_metrics([DependentDummyMetric()]).measure() # All metrics returned (both direct and indirect) - assert len(result.metrics) == 2 + assert len(result.measurements) == 2 # Check direct_metric_names assert "dummy" not in result.direct_metric_names assert "dependent_dummy" in result.direct_metric_names - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["dependent_dummy"].value == 84 # 42 * 2 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["dependent_dummy"].value == 84 # 42 * 2 def test_checkhub_measure_deep_dependency_chain(): @@ -95,16 +96,16 @@ def test_checkhub_measure_deep_dependency_chain(): DummyMetric(42) → DependentDummyMetric(84) → Level2Metric(94) → Level3Metric(8836) """ - result = CheckHub().with_metrics([Level3Metric]).measure() + result = CheckHub().with_metrics([Level3Metric()]).measure() # All 4 metrics returned - assert len(result.metrics) == 4 + assert len(result.measurements) == 4 # Only Level3Metric is direct assert result.direct_metric_names == {"level3"} - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["level3"].value == 8836 # (84 + 10) ** 2 = 94 ** 2 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["level3"].value == 8836 # (84 + 10) ** 2 = 94 ** 2 def test_checkhub_measure_with_provider(): @@ -116,14 +117,14 @@ def test_checkhub_measure_with_provider(): result = ( CheckHub() - .with_metrics([ProviderDummyMetric]) + .with_metrics([ProviderDummyMetric()]) .with_providers([[DummyProvider()]]) .measure() ) - assert len(result.metrics) == 1 - assert result.metrics[0].name == "provider_dummy" - assert result.metrics[0].value == 100 # Value from dummy_provider + assert len(result.measurements) == 1 + assert result.measurements[0].metric.name == "provider_dummy" + assert result.measurements[0].value == 100 # Value from dummy_provider def test_checkhub_measure_with_providers(): @@ -132,22 +133,12 @@ def test_checkhub_measure_with_providers(): result = ( CheckHub() - .with_metrics([ProviderDummyMetric]) + .with_metrics([ProviderDummyMetric()]) .with_providers([[DummyProvider()]]) .measure() ) - assert result.metrics[0].value == 100 # Provider works - - -def test_checkhub_measure_with_config(): - """Test measuring metrics with YAML config.""" - config_path = Path(__file__).parent / "fixtures" / "test_config.yaml" - - result = CheckHub(config_path=config_path).with_metrics([DummyMetric]).measure() - - assert len(result.metrics) == 1 - assert result.metrics[0].value == 200 # From YAML, not default 42 + assert result.measurements[0].value == 100 # Provider works def test_measurement_result_materialize(): @@ -155,7 +146,7 @@ def test_measurement_result_materialize(): captured_output = StringIO() sys.stdout = captured_output - CheckHub().with_metrics([DummyMetric]).measure().materialize( + CheckHub().with_metrics([DummyMetric()]).measure().materialize( ConsoleMaterializer(group_tag_1="env", group_tag_2="team") ) @@ -171,7 +162,7 @@ def test_checkhub_measure_multiple_provider_sets(): result = ( CheckHub() - .with_metrics([DummyMetric]) + .with_metrics([DummyMetric()]) .with_providers( [ [TagProvider(path="/repo1")], @@ -182,8 +173,8 @@ def test_checkhub_measure_multiple_provider_sets(): .measure() ) - assert len(result.metrics) == 3 - paths = {m.tags["path"] for m in result.metrics} + assert len(result.measurements) == 3 + paths = {m.tags["path"] for m in result.measurements} assert paths == {"/repo1", "/repo2", "/repo3"} @@ -193,9 +184,9 @@ def test_checkhub_measure_parallel(): result = ( CheckHub() - .with_metrics([DummyMetric]) + .with_metrics([DummyMetric()]) .with_providers([[TagProvider(path=f"/repo{i}")] for i in range(10)]) .measure(max_workers=4) ) - assert len(result.metrics) == 10 + assert len(result.measurements) == 10 diff --git a/tests/test_hub_execution.py b/tests/test_hub_execution.py index b5f015f..3e131ca 100644 --- a/tests/test_hub_execution.py +++ b/tests/test_hub_execution.py @@ -3,7 +3,7 @@ from typing import Any, ClassVar from checkup.hub import CheckHub -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.provider import Provider from checkup.providers.tags import TagProvider from checkup.types import Context @@ -32,8 +32,11 @@ class DataMetric(Metric): def providers(cls) -> list[type[Provider]]: return [DataProvider] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = context[DataProvider.name]["value"] + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + value = context[DataProvider.name]["value"] + return self.measurement(value=value) class OtherProvider(Provider): @@ -65,8 +68,10 @@ class FailingProviderMetric(Metric): def providers(cls) -> list[type[Provider]]: return [FailingProvider] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 999 + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement(value=999) class DependsOnFailingMetric(Metric): @@ -84,8 +89,11 @@ def depends_on(cls) -> list[type[Metric]]: def providers(cls) -> list[type[Provider]]: return [FailingProvider] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = metrics[FailingProviderMetric].value * 2 + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + base_val = measurements[FailingProviderMetric].value + return self.measurement(value=base_val * 2) class OtherMetric(Metric): @@ -99,8 +107,11 @@ class OtherMetric(Metric): def providers(cls) -> list[type[Provider]]: return [OtherProvider] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = context[OtherProvider.name]["other_value"] + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + value = context[OtherProvider.name]["other_value"] + return self.measurement(value=value) class TestHubExecution: @@ -110,19 +121,19 @@ def test_measure_with_single_provider_set(self): """Test measuring with single provider set.""" result = ( CheckHub() - .with_metrics([DataMetric]) + .with_metrics([DataMetric()]) .with_providers([[DataProvider(value=42)]]) .measure() ) - assert len(result.metrics) == 1 - assert result.metrics[0].value == 42 + assert len(result.measurements) == 1 + assert result.measurements[0].value == 42 def test_measure_with_multiple_provider_sets(self): """Test measuring across multiple provider sets.""" result = ( CheckHub() - .with_metrics([DataMetric]) + .with_metrics([DataMetric()]) .with_providers( [ [DataProvider(value=10)], @@ -133,15 +144,15 @@ def test_measure_with_multiple_provider_sets(self): .measure() ) - assert len(result.metrics) == 3 - values = {m.value for m in result.metrics} + assert len(result.measurements) == 3 + values = {m.value for m in result.measurements} assert values == {10, 20, 30} def test_measure_with_tag_provider(self): """Test that TagProvider merges into metric tags.""" result = ( CheckHub() - .with_metrics([DataMetric]) + .with_metrics([DataMetric()]) .with_providers( [ [DataProvider(value=42), TagProvider(env="prod", team="data")], @@ -150,16 +161,16 @@ def test_measure_with_tag_provider(self): .measure() ) - metric = result.metrics[0] - assert metric.tags["env"] == "prod" - assert metric.tags["team"] == "data" + measurement = result.measurements[0] + assert measurement.tags["env"] == "prod" + assert measurement.tags["team"] == "data" def test_measure_warns_on_missing_providers(self, caplog): """Test that measure() warns when providers are missing.""" import logging with caplog.at_level(logging.WARNING): - CheckHub().with_metrics([DataMetric]).with_providers([[]]).measure() + CheckHub().with_metrics([DataMetric()]).with_providers([[]]).measure() assert "data" in caplog.text.lower() @@ -167,10 +178,10 @@ def test_measure_with_empty_provider_sets_and_no_requirements(self): """Test measuring without providers when none required.""" from fixtures import DummyMetric - result = CheckHub().with_metrics([DummyMetric]).measure() + result = CheckHub().with_metrics([DummyMetric()]).measure() - assert len(result.metrics) == 1 - assert result.metrics[0].value == 42 + assert len(result.measurements) == 1 + assert result.measurements[0].value == 42 def test_measure_skips_only_metrics_with_missing_providers(self, caplog): """Test that metrics with available providers are still calculated when others are missing.""" @@ -180,16 +191,16 @@ def test_measure_skips_only_metrics_with_missing_providers(self, caplog): with caplog.at_level(logging.WARNING): result = ( CheckHub() - .with_metrics([DataMetric, OtherMetric]) + .with_metrics([DataMetric(), OtherMetric()]) .with_providers([[DataProvider(value=42)]]) .measure() ) # DataMetric should be calculated (has its provider) # OtherMetric should be skipped (missing OtherProvider) - assert len(result.metrics) == 1 - assert result.metrics[0].name == "data_metric" - assert result.metrics[0].value == 42 + assert len(result.measurements) == 1 + assert result.measurements[0].metric.name == "data_metric" + assert result.measurements[0].value == 42 # Warning should be logged for the missing provider assert "other" in caplog.text.lower() @@ -203,13 +214,13 @@ def test_measure_skips_dependent_metrics_when_dependency_skipped(self): # If we don't provide IntegrationProvider, both should be skipped result = ( CheckHub() - .with_metrics([IntegrationBaseMetric, IntegrationDerivedMetric]) + .with_metrics([IntegrationBaseMetric(), IntegrationDerivedMetric()]) .with_providers([[]]) # No providers .measure() ) # Both metrics should be skipped - no errors should occur - assert len(result.metrics) == 0 + assert len(result.measurements) == 0 assert ( len(result.errors) == 0 ) # No exceptions from trying to access missing dependency @@ -218,56 +229,63 @@ def test_failed_provider_metric_has_null_value_with_diagnostic(self): """When a provider fails, metrics depending on it have value=None with diagnostic.""" result = ( CheckHub() - .with_metrics([FailingProviderMetric]) + .with_metrics([FailingProviderMetric()]) .with_providers([[FailingProvider()]]) .measure() ) - assert len(result.metrics) == 1 - metric = result.metrics[0] - assert metric.name == "failing_provider_metric" - assert metric.value is None - assert "provider 'failing' failed" in metric.diagnostic + assert len(result.measurements) == 1 + measurement = result.measurements[0] + assert measurement.metric.name == "failing_provider_metric" + assert measurement.value is None + assert "provider 'failing' failed" in measurement.diagnostic def test_failed_provider_does_not_affect_unrelated_metrics(self): """When a provider fails, unrelated metrics are still calculated.""" result = ( CheckHub() - .with_metrics([DataMetric, FailingProviderMetric]) + .with_metrics([DataMetric(), FailingProviderMetric()]) .with_providers([[DataProvider(value=42), FailingProvider()]]) .measure() ) - assert len(result.metrics) == 2 + assert len(result.measurements) == 2 - data_metric = next(m for m in result.metrics if m.name == "data_metric") - assert data_metric.value == 42 + data_measurement = next( + m for m in result.measurements if m.metric.name == "data_metric" + ) + assert data_measurement.value == 42 - failing_metric = next( - m for m in result.metrics if m.name == "failing_provider_metric" + failing_measurement = next( + m for m in result.measurements if m.metric.name == "failing_provider_metric" ) - assert failing_metric.value is None - assert "failed" in failing_metric.diagnostic + assert failing_measurement.value is None + assert "failed" in failing_measurement.diagnostic def test_metric_depending_on_failed_metric_also_fails(self): """When a metric fails due to provider failure, dependents also fail.""" result = ( CheckHub() - .with_metrics([FailingProviderMetric, DependsOnFailingMetric]) + .with_metrics([FailingProviderMetric(), DependsOnFailingMetric()]) .with_providers([[FailingProvider()]]) .measure() ) - assert len(result.metrics) == 2 + assert len(result.measurements) == 2 - base_metric = next( - m for m in result.metrics if m.name == "failing_provider_metric" + base_measurement = next( + m for m in result.measurements if m.metric.name == "failing_provider_metric" ) - assert base_metric.value is None - assert "provider 'failing' failed" in base_metric.diagnostic + assert base_measurement.value is None + assert "provider 'failing' failed" in base_measurement.diagnostic - dependent_metric = next( - m for m in result.metrics if m.name == "depends_on_failing_metric" + dependent_measurement = next( + m + for m in result.measurements + if m.metric.name == "depends_on_failing_metric" + ) + assert dependent_measurement.value is None + assert ( + "metric 'failing_provider_metric' failed" + in dependent_measurement.diagnostic ) - assert dependent_metric.value is None - assert "metric 'failing_provider_metric' failed" in dependent_metric.diagnostic diff --git a/tests/test_hub_validation.py b/tests/test_hub_validation.py index 9cec004..82d4a9b 100644 --- a/tests/test_hub_validation.py +++ b/tests/test_hub_validation.py @@ -2,7 +2,7 @@ from typing import Any, ClassVar -from checkup.metric import Metric +from checkup.metric import Measurement, Metric from checkup.provider import Provider from checkup.types import Context from checkup.validators import validate_providers @@ -37,8 +37,10 @@ class MetricWithProvider(Metric): def providers(cls) -> list[type[Provider]]: return [RequiredProvider] - def calculate(self, context: Context, metrics: dict) -> None: - self.value = 1 + def calculate( + self, context: Context, measurements: dict[type[Metric], Measurement] + ) -> Measurement: + return self.measurement(value=1) class TestProviderValidation: diff --git a/tests/test_integration.py b/tests/test_integration.py index 219c056..20ccc58 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -16,36 +16,36 @@ def test_full_pipeline(): """Test complete pipeline from provider through metric calculation.""" result = ( CheckHub() - .with_metrics([IntegrationBaseMetric]) + .with_metrics([IntegrationBaseMetric()]) .with_providers([[IntegrationProvider()]]) .measure() ) - assert len(result.metrics) == 1 - assert result.metrics[0].value == 25 + assert len(result.measurements) == 1 + assert result.measurements[0].value == 25 def test_full_pipeline_with_both_metrics(): """Test pipeline with dependent metrics.""" result = ( CheckHub() - .with_metrics([IntegrationDerivedMetric]) + .with_metrics([IntegrationDerivedMetric()]) .with_providers([[IntegrationProvider()]]) .measure() ) # Both base and derived metrics should be calculated - assert len(result.metrics) == 2 - metrics_by_name = {m.name: m for m in result.metrics} - assert metrics_by_name["base_metric"].value == 25 - assert metrics_by_name["derived_metric"].value == 50 # 25 * 2 + assert len(result.measurements) == 2 + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["base_metric"].value == 25 + assert measurements_by_name["derived_metric"].value == 50 # 25 * 2 def test_multi_provider_set_pipeline(): """Test pipeline across multiple provider sets.""" result = ( CheckHub() - .with_metrics([PathMetric]) + .with_metrics([PathMetric()]) .with_providers( [ [PathLengthProvider(path="/short"), TagProvider(name="short")], @@ -58,8 +58,8 @@ def test_multi_provider_set_pipeline(): .measure() ) - assert len(result.metrics) == 2 + assert len(result.measurements) == 2 - metrics_by_name = {m.tags["name"]: m for m in result.metrics} - assert metrics_by_name["short"].value == 6 # len("/short") - assert metrics_by_name["long"].value == 17 # len("/much/longer/path") + measurements_by_name = {m.tags["name"]: m for m in result.measurements} + assert measurements_by_name["short"].value == 6 # len("/short") + assert measurements_by_name["long"].value == 17 # len("/much/longer/path") diff --git a/tests/test_metric.py b/tests/test_metric.py index d4e1b51..4b18732 100644 --- a/tests/test_metric.py +++ b/tests/test_metric.py @@ -22,7 +22,6 @@ def test_dummy_metric_with_explicit_value(): assert metric.name == "dummy" assert metric.description == "Test metric" assert metric.unit == "count" - assert metric.value is None assert metric.expected_value == 42 @@ -31,7 +30,6 @@ def test_dummy_metric_with_default_value(): metric = DummyMetric() assert metric.expected_value == 42 - assert metric.value is None # ============================================================================= @@ -39,22 +37,21 @@ def test_dummy_metric_with_default_value(): # ============================================================================= -def test_dummy_metric_calculate_sets_value(empty_context): - """Test that DummyMetric.calculate() sets value correctly.""" +def test_dummy_metric_calculate_returns_measurement(empty_context): + """Test that DummyMetric.calculate() returns a Measurement.""" metric = DummyMetric(expected_value=100) - assert metric.value is None + measurement = metric.calculate(context=empty_context, measurements={}) - metric.calculate(context=empty_context, metrics={}) - - assert metric.value == 100 + assert measurement.value == 100 + assert measurement.metric.name == "dummy" def test_dummy_metric_calculate_uses_fixture(dummy_metric, empty_context): """Test calculate using pytest fixtures.""" - dummy_metric.calculate(context=empty_context, metrics={}) + measurement = dummy_metric.calculate(context=empty_context, measurements={}) - assert dummy_metric.value == 42 + assert measurement.value == 42 # ============================================================================= @@ -88,7 +85,6 @@ def test_metric_pydantic_model_dump(): # Instance fields are in model_dump data = metric.model_dump() assert data["expected_value"] == 50 - assert data["value"] is None assert "name" not in data # ClassVar not included in dump @@ -104,23 +100,27 @@ def test_dependent_metric_depends_on(): assert deps == [DummyMetric] -def test_dependent_metric_calculate(dummy_metric_with_value): +def test_dependent_metric_calculate(dummy_measurement_with_value): """Test that DependentDummyMetric uses dependency value.""" dependent = DependentDummyMetric() - dependent.calculate(context={}, metrics={DummyMetric: dummy_metric_with_value}) + measurement = dependent.calculate( + context={}, measurements={DummyMetric: dummy_measurement_with_value} + ) - assert dependent.value == 20 # 10 * 2 + assert measurement.value == 20 # 10 * 2 def test_dependent_metric_calculate_custom_base(empty_context): """Test calculation with custom base value.""" base_metric = DummyMetric(expected_value=25) - base_metric.calculate(context=empty_context, metrics={}) + base_measurement = base_metric.calculate(context=empty_context, measurements={}) dependent = DependentDummyMetric() - dependent.calculate(context=empty_context, metrics={DummyMetric: base_metric}) + measurement = dependent.calculate( + context=empty_context, measurements={DummyMetric: base_measurement} + ) - assert dependent.value == 50 # 25 * 2 + assert measurement.value == 50 # 25 * 2 # ============================================================================= @@ -156,6 +156,6 @@ def test_provider_dummy_metric_calculate(): context = {DummyProvider.name: provider.provide()} metric = ProviderDummyMetric() - metric.calculate(context=context, metrics={}) + measurement = metric.calculate(context=context, measurements={}) - assert metric.value == 100 + assert measurement.value == 100 From ef95b82ecce2dfc292f90ede2a3a3f566a2e98d2 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Wed, 8 Apr 2026 16:55:29 +0200 Subject: [PATCH 07/21] update --- checkup.schema.json | 632 ++++++++++++++++++ checkup.yaml | 16 + plugins/checkup-airflow/pyproject.toml | 3 + plugins/checkup-bitbucket/pyproject.toml | 3 + plugins/checkup-conveyor/pyproject.toml | 8 + plugins/checkup-dbt/pyproject.toml | 27 + plugins/checkup-git/pyproject.toml | 7 + plugins/checkup-github/pyproject.toml | 3 + plugins/checkup-gitlab/pyproject.toml | 3 + plugins/checkup-python/pyproject.toml | 4 + .../src/checkup_python/metrics/__init__.py | 3 +- src/checkup/cli/__init__.py | 27 + src/checkup/cli/commands/__init__.py | 17 + src/checkup/cli/commands/check.py | 48 ++ src/checkup/cli/commands/config.py | 23 + src/checkup/cli/commands/init.py | 23 + src/checkup/cli/commands/run.py | 58 ++ src/checkup/cli/commands/schema.py | 28 + src/checkup/cli/config_wizard.py | 403 +++++++++++ src/checkup/cli/executor.py | 154 +++++ src/checkup/cli/utils.py | 69 ++ src/checkup/configuration/__init__.py | 18 + src/checkup/configuration/env.py | 135 ++++ src/checkup/configuration/io.py | 212 ++++++ src/checkup/configuration/models.py | 41 ++ src/checkup/configuration/schema.py | 246 +++++++ src/checkup/registry/__init__.py | 13 + src/checkup/registry/discovery.py | 190 ++++++ tests/test_cli_configuration.py | 288 ++++++++ tests/test_registry.py | 187 ++++++ 30 files changed, 2888 insertions(+), 1 deletion(-) create mode 100644 checkup.schema.json create mode 100644 checkup.yaml create mode 100644 src/checkup/cli/__init__.py create mode 100644 src/checkup/cli/commands/__init__.py create mode 100644 src/checkup/cli/commands/check.py create mode 100644 src/checkup/cli/commands/config.py create mode 100644 src/checkup/cli/commands/init.py create mode 100644 src/checkup/cli/commands/run.py create mode 100644 src/checkup/cli/commands/schema.py create mode 100644 src/checkup/cli/config_wizard.py create mode 100644 src/checkup/cli/executor.py create mode 100644 src/checkup/cli/utils.py create mode 100644 src/checkup/configuration/__init__.py create mode 100644 src/checkup/configuration/env.py create mode 100644 src/checkup/configuration/io.py create mode 100644 src/checkup/configuration/models.py create mode 100644 src/checkup/configuration/schema.py create mode 100644 src/checkup/registry/__init__.py create mode 100644 src/checkup/registry/discovery.py create mode 100644 tests/test_cli_configuration.py create mode 100644 tests/test_registry.py diff --git a/checkup.schema.json b/checkup.schema.json new file mode 100644 index 0000000..41d1c87 --- /dev/null +++ b/checkup.schema.json @@ -0,0 +1,632 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://checkup.dev/schemas/checkup.yaml.json", + "title": "Checkup Configuration", + "description": "Configuration file for checkup data product health metrics", + "type": "object", + "properties": { + "tags": { + "type": "object", + "description": "Tags to identify the data product (e.g., product, team)", + "additionalProperties": { + "type": "string" + } + }, + "providers": { + "type": "array", + "description": "Data providers for context enrichment", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "name": { + "const": "airflow" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "bitbucket" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "conveyor" + }, + "project_name": { + "type": "string" + }, + "api_key": { + "type": "string" + }, + "environment_name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt" + }, + "manifest_path": { + "type": "string", + "default": null + }, + "dbt_project_dir": { + "type": "string", + "default": null + }, + "profiles_dir": { + "type": "string", + "default": null + }, + "verbose": { + "type": "boolean", + "default": false + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "git" + }, + "repo_path": { + "type": "string", + "default": "." + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "github" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "gitlab" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + ] + } + }, + "metrics": { + "type": "array", + "description": "Metrics to calculate", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "name": { + "const": "conveyor_is_dirty_deployment" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "conveyor_last_deployment_time" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "conveyor_last_run_status" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_column_test_coverage" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_column_tests" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_columns" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_columns_with_description" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_columns_without_description" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_data_tests" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_flagged_packages" + }, + "flagged_packages": { + "items": { + "type": "string" + }, + "title": "Flagged Packages", + "type": "array" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_models" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_models_not_adhering_to_naming_convention" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_models_with_description" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_models_without_description" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_output_columns_without_data_type" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_output_models" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_output_models_with_description" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_output_models_without_contracts" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_output_models_without_description" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_profile_host" + }, + "profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Profile" + }, + "target": { + "title": "Target", + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_supported_version" + }, + "min_version": { + "title": "Min Version", + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_tested_columns" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_tests" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_unit_tests" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "dbt_version" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "git_days_since_last_update" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "git_tracked_file_count" + }, + "pattern": { + "default": "*", + "title": "Pattern", + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "python_version" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "const": "python_version_check" + }, + "min_version": { + "title": "Min Version", + "type": "string" + }, + "max_version": { + "title": "Max Version", + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + ] + } + }, + "materializer": { + "type": "object", + "description": "Output materializer configuration", + "properties": { + "type": { + "type": "string", + "enum": [ + "console", + "csv", + "html", + "sqlalchemy" + ] + }, + "include_indirect": { + "default": false, + "title": "Include Indirect", + "type": "boolean" + }, + "group_tag_1": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Group Tag 1" + }, + "group_tag_2": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Group Tag 2" + }, + "output_path": { + "format": "path", + "title": "Output Path", + "type": "string" + }, + "connection_url": { + "format": "password", + "title": "Connection Url", + "type": "string", + "writeOnly": true + }, + "table_name": { + "default": "metrics", + "title": "Table Name", + "type": "string" + }, + "table_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Table Schema" + }, + "connect_args": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Connect Args" + }, + "expand_tags": { + "default": false, + "title": "Expand Tags", + "type": "boolean" + }, + "batch_size": { + "default": 1000, + "title": "Batch Size", + "type": "integer" + } + }, + "required": [ + "type" + ] + } + }, + "additionalProperties": false +} diff --git a/checkup.yaml b/checkup.yaml new file mode 100644 index 0000000..85e4ca8 --- /dev/null +++ b/checkup.yaml @@ -0,0 +1,16 @@ +# yaml-language-server: $schema=checkup.schema.json + +tags: + project: checkup + +providers: +- name: git + +metrics: +- name: git_days_since_last_update +- name: git_tracked_file_count + pattern: src/checkup/* +- name: python_version +- name: python_version_check + min_version: '3.13' + max_version: '3.15' diff --git a/plugins/checkup-airflow/pyproject.toml b/plugins/checkup-airflow/pyproject.toml index 364d26b..cff6a8f 100644 --- a/plugins/checkup-airflow/pyproject.toml +++ b/plugins/checkup-airflow/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +airflow = "checkup_airflow:AirflowProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-bitbucket/pyproject.toml b/plugins/checkup-bitbucket/pyproject.toml index 5087151..47e5c08 100644 --- a/plugins/checkup-bitbucket/pyproject.toml +++ b/plugins/checkup-bitbucket/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +bitbucket = "checkup_bitbucket:BitbucketProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-conveyor/pyproject.toml b/plugins/checkup-conveyor/pyproject.toml index b734070..7293e6d 100644 --- a/plugins/checkup-conveyor/pyproject.toml +++ b/plugins/checkup-conveyor/pyproject.toml @@ -12,6 +12,14 @@ dependencies = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +conveyor = "checkup_conveyor:ConveyorProvider" + +[project.entry-points."checkup.metrics"] +conveyor_last_deployment_time = "checkup_conveyor.conveyor_metric:ConveyorLastDeploymentTime" +conveyor_is_dirty_deployment = "checkup_conveyor.conveyor_metric:ConveyorIsDirtyDeployment" +conveyor_last_run_status = "checkup_conveyor.conveyor_metric:ConveyorLastRunStatus" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-dbt/pyproject.toml b/plugins/checkup-dbt/pyproject.toml index aceda83..6656c70 100644 --- a/plugins/checkup-dbt/pyproject.toml +++ b/plugins/checkup-dbt/pyproject.toml @@ -18,6 +18,33 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +dbt = "checkup_dbt:DbtManifestProvider" + +[project.entry-points."checkup.metrics"] +dbt_models = "checkup_dbt:DbtModelsMetric" +dbt_columns = "checkup_dbt:DbtColumnsMetric" +dbt_tests = "checkup_dbt:DbtTestsMetric" +dbt_models_with_description = "checkup_dbt:DbtModelsWithDescriptionMetric" +dbt_models_without_description = "checkup_dbt:DbtModelsWithoutDescriptionMetric" +dbt_columns_with_description = "checkup_dbt:DbtColumnsWithDescriptionMetric" +dbt_columns_without_description = "checkup_dbt:DbtColumnsWithoutDescriptionMetric" +dbt_unit_tests = "checkup_dbt:DbtUnitTestsMetric" +dbt_data_tests = "checkup_dbt:DbtDataTestsMetric" +dbt_column_tests = "checkup_dbt:DbtColumnTestsMetric" +dbt_tested_columns = "checkup_dbt:DbtTestedColumnsMetric" +dbt_column_test_coverage = "checkup_dbt:DbtColumnTestCoverageMetric" +dbt_output_models = "checkup_dbt:DbtOutputModelsMetric" +dbt_output_models_with_description = "checkup_dbt:DbtOutputModelsWithDescriptionMetric" +dbt_output_models_without_description = "checkup_dbt:DbtOutputModelsWithoutDescriptionMetric" +dbt_output_models_without_contracts = "checkup_dbt:DbtOutputModelsWithoutContractsMetric" +dbt_output_columns_without_data_type = "checkup_dbt:DbtOutputColumnsWithoutDataTypeMetric" +dbt_flagged_packages = "checkup_dbt:DbtFlaggedPackagesMetric" +dbt_profile_host = "checkup_dbt:DbtProfileHostMetric" +dbt_models_not_adhering_to_naming_convention = "checkup_dbt:DbtModelsNotAdheringToNamingConventionMetric" +dbt_supported_version = "checkup_dbt:DbtSupportedVersionMetric" +dbt_version = "checkup_dbt:DbtVersionMetric" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-git/pyproject.toml b/plugins/checkup-git/pyproject.toml index dccc8d5..37e2bce 100644 --- a/plugins/checkup-git/pyproject.toml +++ b/plugins/checkup-git/pyproject.toml @@ -16,6 +16,13 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +git = "checkup_git:GitProvider" + +[project.entry-points."checkup.metrics"] +git_days_since_last_update = "checkup_git:GitDaysSinceLastUpdateMetric" +git_tracked_file_count = "checkup_git:GitTrackedFileCountMetric" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-github/pyproject.toml b/plugins/checkup-github/pyproject.toml index f7dd520..15bd95e 100644 --- a/plugins/checkup-github/pyproject.toml +++ b/plugins/checkup-github/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +github = "checkup_github:GitHubProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-gitlab/pyproject.toml b/plugins/checkup-gitlab/pyproject.toml index bf7e948..1fe49f9 100644 --- a/plugins/checkup-gitlab/pyproject.toml +++ b/plugins/checkup-gitlab/pyproject.toml @@ -16,6 +16,9 @@ dev = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.providers"] +gitlab = "checkup_gitlab:GitLabProvider" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-python/pyproject.toml b/plugins/checkup-python/pyproject.toml index cb8faba..8c40c9d 100644 --- a/plugins/checkup-python/pyproject.toml +++ b/plugins/checkup-python/pyproject.toml @@ -11,6 +11,10 @@ dependencies = [ [tool.uv.sources] checkup = { workspace = true } +[project.entry-points."checkup.metrics"] +python_version = "checkup_python.metrics:PythonVersionMetric" +python_version_check = "checkup_python.metrics:PythonVersionCheckMetric" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/plugins/checkup-python/src/checkup_python/metrics/__init__.py b/plugins/checkup-python/src/checkup_python/metrics/__init__.py index 85b2011..9e86bc1 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/__init__.py +++ b/plugins/checkup-python/src/checkup_python/metrics/__init__.py @@ -1,3 +1,4 @@ from checkup_python.metrics.version import PythonVersionMetric +from checkup_python.metrics.version_check import PythonVersionCheckMetric -__all__ = ["PythonVersionMetric"] +__all__ = ["PythonVersionMetric", "PythonVersionCheckMetric"] diff --git a/src/checkup/cli/__init__.py b/src/checkup/cli/__init__.py new file mode 100644 index 0000000..6b3be8c --- /dev/null +++ b/src/checkup/cli/__init__.py @@ -0,0 +1,27 @@ +""" +Checkup CLI application. +""" + +import typer + +from checkup.cli.commands import check, config, init, run, schema + +app = typer.Typer( + name="checkup", + help="CheckUp - Computational governance framework for measuring data product health", + no_args_is_help=True, +) + +app.command()(check) +app.command()(run) +app.command()(init) +app.command()(config) +app.command()(schema) + + +def main() -> None: + """ + CLI entry point. + """ + + app() diff --git a/src/checkup/cli/commands/__init__.py b/src/checkup/cli/commands/__init__.py new file mode 100644 index 0000000..55a6a66 --- /dev/null +++ b/src/checkup/cli/commands/__init__.py @@ -0,0 +1,17 @@ +""" +CLI commands. +""" + +from checkup.cli.commands.check import check +from checkup.cli.commands.config import config +from checkup.cli.commands.init import init +from checkup.cli.commands.run import run +from checkup.cli.commands.schema import schema + +__all__ = [ + "check", + "config", + "init", + "run", + "schema", +] diff --git a/src/checkup/cli/commands/check.py b/src/checkup/cli/commands/check.py new file mode 100644 index 0000000..6039ec3 --- /dev/null +++ b/src/checkup/cli/commands/check.py @@ -0,0 +1,48 @@ +""" +Check command. Run metrics locally. +""" + +import logging +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.executor import execute_checkup +from checkup.cli.utils import apply_cli_overrides +from checkup.configuration import load_config + + +def check( + config: Annotated[ + Path | None, + typer.Option("--config", "-c", help="Path to config file"), + ] = None, + tag: Annotated[ + list[str] | None, + typer.Option("--tag", "-t", help="Set tags (key=value)"), + ] = None, + provider: Annotated[ + list[str] | None, + typer.Option("--provider", "-p", help="Set providers (name or name:key=value)"), + ] = None, + metric: Annotated[ + list[str] | None, + typer.Option("--metric", "-m", help="Set metrics (name or name:key=value)"), + ] = None, + verbose: Annotated[ + bool, + typer.Option("--verbose", "-v", help="Verbose output"), + ] = False, +) -> None: + """ + Run metrics and show results. + """ + + if verbose: + logging.basicConfig(level=logging.DEBUG) + + cfg = load_config(config_path=config) + cfg = apply_cli_overrides(cfg, tag, provider, metric) + + execute_checkup(cfg, materializer="console") diff --git a/src/checkup/cli/commands/config.py b/src/checkup/cli/commands/config.py new file mode 100644 index 0000000..e27f25e --- /dev/null +++ b/src/checkup/cli/commands/config.py @@ -0,0 +1,23 @@ +""" +Config command. Modify an existing config file. +""" + +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.config_wizard import edit_config + + +def config( + config_path: Annotated[ + Path | None, + typer.Option("--config", "-c", help="Path to config file"), + ] = None, +) -> None: + """ + Modify the checkup.yaml config file. + """ + + edit_config(config_path=config_path) diff --git a/src/checkup/cli/commands/init.py b/src/checkup/cli/commands/init.py new file mode 100644 index 0000000..a224cb1 --- /dev/null +++ b/src/checkup/cli/commands/init.py @@ -0,0 +1,23 @@ +""" +Init command. Create a new config file. +""" + +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.config_wizard import create_config + + +def init( + output: Annotated[ + Path | None, + typer.Option("--output", "-o", help="Output path for config file"), + ] = None, +) -> None: + """ + Create a checkup.yaml config file. + """ + + create_config(output_path=output) diff --git a/src/checkup/cli/commands/run.py b/src/checkup/cli/commands/run.py new file mode 100644 index 0000000..1d1ca49 --- /dev/null +++ b/src/checkup/cli/commands/run.py @@ -0,0 +1,58 @@ +""" +Run command. Run metrics and materialize results. +""" + +import logging +from pathlib import Path +from typing import Annotated + +import typer + +from checkup.cli.executor import execute_checkup +from checkup.cli.utils import apply_cli_overrides +from checkup.configuration import load_config + + +def run( + config: Annotated[ + Path | None, + typer.Option("--config", "-c", help="Path to config file"), + ] = None, + tag: Annotated[ + list[str] | None, + typer.Option("--tag", "-t", help="Set tags (key=value)"), + ] = None, + provider: Annotated[ + list[str] | None, + typer.Option("--provider", "-p", help="Set providers (name or name:key=value)"), + ] = None, + metric: Annotated[ + list[str] | None, + typer.Option("--metric", "-m", help="Set metrics (name or name:key=value)"), + ] = None, + materializer: Annotated[ + str | None, + typer.Option( + "--materializer", help="Set materializer (type or type:key=value)" + ), + ] = None, + dry_run: Annotated[ + bool, + typer.Option("--dry-run", help="Don't materialize, just print (same as check)"), + ] = False, + verbose: Annotated[ + bool, + typer.Option("--verbose", "-v", help="Verbose output"), + ] = False, +) -> None: + """ + Run metrics and materialize results. + """ + + if verbose: + logging.basicConfig(level=logging.DEBUG) + + cfg = load_config(config_path=config) + cfg = apply_cli_overrides(cfg, tag, provider, metric) + + execute_checkup(cfg, materializer="console" if dry_run else materializer) diff --git a/src/checkup/cli/commands/schema.py b/src/checkup/cli/commands/schema.py new file mode 100644 index 0000000..7fbe768 --- /dev/null +++ b/src/checkup/cli/commands/schema.py @@ -0,0 +1,28 @@ +""" +Schema command. Generate JSON schema for checkup.yaml. +""" + +from pathlib import Path +from typing import Annotated + +import typer +from rich.console import Console + +from checkup.configuration.schema import write_schema + +console = Console() + + +def schema( + output: Annotated[ + Path | None, + typer.Option("--output", "-o", help="Output path for schema file"), + ] = None, +) -> None: + """ + Generate JSON schema for checkup.yaml. + """ + + path = output or Path.cwd() / "checkup.schema.json" + write_schema(path) + console.print(f"[green]Schema written to {path}[/green]") diff --git a/src/checkup/cli/config_wizard.py b/src/checkup/cli/config_wizard.py new file mode 100644 index 0000000..351200a --- /dev/null +++ b/src/checkup/cli/config_wizard.py @@ -0,0 +1,403 @@ +""" +Interactive config generation and editing. +""" + +from pathlib import Path +from typing import TYPE_CHECKING + +import yaml +from rich.console import Console + +from checkup.configuration.io import CONFIG_FILENAME +from checkup.configuration.schema import write_schema +from checkup.registry import get_registry + +if TYPE_CHECKING: + import questionary + + from checkup.configuration import CheckupConfig + from checkup.registry import PluginRegistry + +console = Console(markup=False, highlight=False) + + +def _get_questionary() -> "questionary": + """ + Lazy import questionary to avoid slow startup. + """ + + import questionary + + return questionary + + +def create_config(output_path: Path | None = None) -> None: + """ + Interactively create a new config file. + """ + + path = output_path or Path.cwd() / CONFIG_FILENAME + + if path.exists(): + overwrite = ( + _get_questionary() + .confirm(f"{path} exists. Overwrite?", default=False) + .ask() + ) + if overwrite is None or not overwrite: + if overwrite is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + + registry = get_registry() + config: dict = {} + + # Tags + console.print("\n[bold]Tags[/bold]", markup=True) + console.print( + "Tags identify your data product (e.g., product=my-product, team=analytics)" + ) + tags = _collect_tags({}) + if tags is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if tags: + config["tags"] = tags + + # Providers + console.print("\n[bold]Providers[/bold]", markup=True) + provider_names = _select_multiple(registry.list_provider_names(), [], "providers") + if provider_names is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if provider_names: + config["providers"] = [{"name": p} for p in provider_names] + + # Metrics + console.print("\n[bold]Metrics[/bold]", markup=True) + with console.status("Loading metrics..."): + available_metrics = registry.list_compatible_metric_names(provider_names or []) + metric_names = _select_multiple(available_metrics, [], "metrics") + if metric_names is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if metric_names: + config["metrics"] = [{"name": m} for m in metric_names] + + # Materializer + console.print("\n[bold]Materializer[/bold]", markup=True) + mat = _select_materializer(None, registry) + if mat is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if mat != "console": + config["materializer"] = {"type": mat} + + # Write + _write_config(path, config) + + +def edit_config(config_path: Path | None = None) -> None: + """ + Interactively edit an existing config file. + """ + + from checkup.configuration import load_config + + path = config_path or Path.cwd() / CONFIG_FILENAME + + if not path.exists(): + console.print(f"[red]Config file not found: {path}[/red]", markup=True) + console.print("Run [bold]checkup init[/bold] to create one.", markup=True) + return + + cfg = load_config(config_path=path) + registry = get_registry() + + console.print(f"[bold]Editing {path}[/bold]\n", markup=True) + _show_current_config(cfg) + + new_config: dict = {} + + # Build lookup for existing configs + existing_provider_configs = {p.name: p.config for p in cfg.providers} + existing_metric_configs = {m.name: m.config for m in cfg.metrics} + + # Tags + edit_tags = _get_questionary().confirm("Edit tags?", default=False).ask() + if edit_tags is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if edit_tags: + tags = _edit_tags(dict(cfg.tags)) + if tags is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if tags: + new_config["tags"] = tags + elif cfg.tags: + new_config["tags"] = dict(cfg.tags) + + # Providers + current_provider_names = [p.name for p in cfg.providers] + edit_providers = _get_questionary().confirm("Edit providers?", default=False).ask() + if edit_providers is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if edit_providers: + provider_names = _select_multiple( + registry.list_provider_names(), + current_provider_names, + "providers", + ) + if provider_names is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if provider_names: + # Preserve existing config for re-selected providers + new_config["providers"] = [ + {"name": p, **existing_provider_configs.get(p, {})} + for p in provider_names + ] + else: + provider_names = current_provider_names + if provider_names: + new_config["providers"] = [ + {"name": p.name, **p.config} for p in cfg.providers + ] + + # Metrics + current_metric_names = [m.name for m in cfg.metrics] + with console.status("Loading metrics..."): + available_metrics = registry.list_compatible_metric_names(provider_names) + edit_metrics = _get_questionary().confirm("Edit metrics?", default=False).ask() + if edit_metrics is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if edit_metrics: + metric_names = _select_multiple( + available_metrics, + current_metric_names, + "metrics", + ) + if metric_names is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if metric_names: + # Preserve existing config for re-selected metrics + new_config["metrics"] = [ + {"name": m, **existing_metric_configs.get(m, {})} for m in metric_names + ] + elif current_metric_names: + new_config["metrics"] = [ + {"name": m.name, **m.config} + for m in cfg.metrics + if m.name in available_metrics + ] + + # Materializer + edit_mat = _get_questionary().confirm("Edit materializer?", default=False).ask() + if edit_mat is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + if edit_mat: + mat = _select_materializer( + cfg.materializer.type if cfg.materializer else None, + registry, + ) + if mat is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + new_config["materializer"] = {"type": mat} + elif cfg.materializer: + new_config["materializer"] = {"type": cfg.materializer.type} + + _write_config(path, new_config) + + +def _show_current_config(cfg: "CheckupConfig") -> None: + """ + Display current configuration. + """ + + console.print("[bold]Current configuration:[/bold]", markup=True) + console.print(f" Tags: {dict(cfg.tags) or '(none)'}") + console.print(f" Providers: {[p.name for p in cfg.providers] or '(none)'}") + console.print(f" Metrics: {[m.name for m in cfg.metrics] or '(none)'}") + mat = cfg.materializer.type if cfg.materializer else "(none)" + console.print(f" Materializer: {mat}") + console.print() + + +def _collect_tags(existing: dict) -> dict | None: + """ + Collect tags interactively. Returns None if cancelled. + """ + + tags = dict(existing) + + while True: + tag = ( + _get_questionary() + .text( + "Add tag (key=value, or empty to finish):", + ) + .ask() + ) + + if tag is None: + return None + if not tag: + break + if "=" in tag: + key, value = tag.split("=", 1) + tags[key.strip()] = value.strip() + + return tags + + +def _edit_tags(tags: dict) -> dict | None: + """ + Edit tags interactively. Returns None if cancelled. + """ + + console.print(f"Current tags: {tags or '(none)'}") + + while True: + action = ( + _get_questionary() + .select( + "Action:", + choices=["done", "add", "remove"], + ) + .ask() + ) + + if action is None: + return None + if action == "done": + break + elif action == "add": + tag = _get_questionary().text("Tag (key=value):").ask() + if tag is None: + return None + if tag and "=" in tag: + key, value = tag.split("=", 1) + tags[key.strip()] = value.strip() + elif action == "remove" and tags: + key = ( + _get_questionary() + .select( + "Key to remove:", + choices=list(tags.keys()), + ) + .ask() + ) + if key is None: + return None + if key: + tags.pop(key, None) + + return tags + + +def _select_multiple( + available: list[str], + selected: list[str], + item_type: str, +) -> list[str]: + """ + Select multiple items with fuzzy search. + """ + + if not available: + console.print( + f"[yellow]No {item_type} found. Install checkup plugins.[/yellow]", + markup=True, + ) + return [] + + # Show current state + selected_count = len([s for s in selected if s in available]) + console.print(f"Currently selected: {selected_count}/{len(available)} {item_type}") + + choices = [ + _get_questionary().Choice(name, checked=name in selected) + for name in sorted(available) + ] + + result = ( + _get_questionary() + .checkbox( + f"Select {item_type}:", + choices=choices, + use_search_filter=True, + use_jk_keys=False, + instruction="(↑↓ navigate, space toggle, type to filter, enter confirm)", + ) + .ask() + ) + + return result or [] + + +def _select_materializer(current: str | None, registry: "PluginRegistry") -> str | None: + """ + Select materializer interactively. + """ + + available = registry.list_materializer_names() + + if not available: + return "console" + + default = ( + current if current in available else (available[0] if available else "console") + ) + + return ( + _get_questionary() + .select( + "Materializer type:", + choices=available, + default=default, + use_search_filter=True, + use_jk_keys=False, + ) + .ask() + ) + + +def _write_config(path: Path, config: dict) -> None: + """ + Write config to file with empty lines between sections. + Also generates the JSON schema file. + """ + + console.print(f"\n[bold]Writing config to {path}[/bold]", markup=True) + + # Write sections separately with blank lines between them + lines = ["# yaml-language-server: $schema=checkup.schema.json"] + + for key in ["tags", "providers", "metrics", "materializer"]: + if key in config: + lines.append("") + lines.append( + yaml.dump( + {key: config[key]}, default_flow_style=False, sort_keys=False + ).rstrip() + ) + + with open(path, "w") as f: + f.write("\n".join(lines)) + f.write("\n") + + # Generate schema file + schema_path = path.parent / "checkup.schema.json" + write_schema(schema_path) + console.print(f"[green]Schema written to {schema_path}[/green]", markup=True) + + console.print("[green]Done![/green]", markup=True) + console.print( + "Run [bold]checkup check[/bold] to test your configuration.", markup=True + ) diff --git a/src/checkup/cli/executor.py b/src/checkup/cli/executor.py new file mode 100644 index 0000000..6dcea94 --- /dev/null +++ b/src/checkup/cli/executor.py @@ -0,0 +1,154 @@ +""" +Execute checkhub from CLI configuration. +""" + +import logging +from typing import TYPE_CHECKING + +from rich.console import Console + +from checkup.configuration import CheckupConfig +from checkup.hub import CheckHub +from checkup.materializers import ConsoleMaterializer +from checkup.providers.tags import TagProvider +from checkup.registry import get_registry + +if TYPE_CHECKING: + from checkup.materializers import Materializer + from checkup.metric import Metric + from checkup.provider import Provider + +logger = logging.getLogger(__name__) +console = Console() + + +def execute_checkup(config: CheckupConfig, materializer: str | None = None) -> None: + """ + Execute checkup with the given configuration. + + Args: + config: Loaded checkup configuration + materializer: Override materializer type (e.g., "console") + """ + + registry = get_registry() + + providers = _resolve_providers(config, registry) + if not providers: + console.print("[yellow]No providers configured[/yellow]") + return + + metrics = _resolve_metrics(config, registry) + if not metrics: + console.print("[yellow]No metrics configured[/yellow]") + return + + materializer = _resolve_materializer(config, registry, materializer) + + console.print(f"[blue]Running {len(metrics)} metrics...[/blue]") + + result = CheckHub().with_metrics(metrics).with_providers([providers]).measure() + + if result.errors: + for _, error in result.errors: + console.print(f"[red]Error: {error}[/red]") + + result.materialize(materializer) + + +def _resolve_providers( + config: CheckupConfig, + registry: "PluginRegistry", +) -> list["Provider"]: + """ + Resolve provider configs to provider instances. + """ + + providers: list[Provider] = [] + + if config.tags: + providers.append(TagProvider(**config.tags)) + + for provider_config in config.providers: + provider_cls = registry.get_provider(provider_config.name) + if provider_cls is None: + console.print(f"[yellow]Unknown provider: {provider_config.name}[/yellow]") + continue + + try: + provider = provider_cls(**provider_config.config) + providers.append(provider) + except Exception as e: + console.print( + f"[red]Failed to instantiate provider {provider_config.name}: {e}[/red]" + ) + + return providers + + +def _resolve_metrics( + config: CheckupConfig, + registry: "PluginRegistry", +) -> list[type["Metric"]]: + """ + Resolve metric configs to metric classes. + + If a metric has config, a subclass is created with those defaults. + """ + from checkup.metric import create_configured_metric + + metrics: list[type[Metric]] = [] + + for metric_config in config.metrics: + metric_cls = registry.get_metric(metric_config.name) + if metric_cls is None: + console.print(f"[yellow]Unknown metric: {metric_config.name}[/yellow]") + continue + + if metric_config.config: + metric_cls = create_configured_metric(metric_cls, metric_config.config) + + metrics.append(metric_cls) + + return metrics + + +def _resolve_materializer( + config: CheckupConfig, + registry: "PluginRegistry", + override: str | None = None, +) -> "Materializer": + """ + Resolve materializer config to materializer instance. + """ + + # Determine type and config + if override: + mat_type = override + mat_config = {} + elif config.materializer: + mat_type = config.materializer.type + mat_config = config.materializer.config + else: + mat_type = "console" + mat_config = {} + + # Get materializer class + materializer_cls = registry.get_materializer(mat_type) + + if materializer_cls is None: + # Fallback to built-in console + console.print( + f"[yellow]Unknown materializer: {mat_type}, using console[/yellow]" + ) + return ConsoleMaterializer() + + try: + return materializer_cls(**mat_config) + except Exception as e: + console.print(f"[red]Failed to instantiate materializer {mat_type}: {e}[/red]") + return ConsoleMaterializer() + + +# Import for type hints +from checkup.registry.discovery import PluginRegistry # noqa: E402 diff --git a/src/checkup/cli/utils.py b/src/checkup/cli/utils.py new file mode 100644 index 0000000..e2e5da4 --- /dev/null +++ b/src/checkup/cli/utils.py @@ -0,0 +1,69 @@ +""" +CLI utility functions. +""" + +from checkup.configuration import CheckupConfig, MetricConfig, ProviderConfig + + +def apply_cli_overrides( + cfg: CheckupConfig, + tags: list[str] | None, + providers: list[str] | None, + metrics: list[str] | None, +) -> CheckupConfig: + """ + Apply CLI flag overrides to config. + + When CLI arguments are provided, they replace the config file values + """ + + if tags: + new_tags = {} + for t in tags: + if "=" in t: + key, value = t.split("=", 1) + new_tags[key] = value + else: + new_tags = dict(cfg.tags) + + if providers: + new_providers = [] + for p in providers: + name, config = parse_cli_item(p) + new_providers.append(ProviderConfig(name=name, config=config)) + else: + new_providers = list(cfg.providers) + + if metrics: + new_metrics = [] + for m in metrics: + name, config = parse_cli_item(m) + new_metrics.append(MetricConfig(name=name, config=config)) + else: + new_metrics = list(cfg.metrics) + + return CheckupConfig( + tags=new_tags, + providers=new_providers, + metrics=new_metrics, + materializer=cfg.materializer, + ) + + +def parse_cli_item(item: str) -> tuple[str, dict]: + """ + Parse CLI item like 'name' or 'name:key=value,key2=value2'. + """ + + if ":" not in item: + return item, {} + + name, config_str = item.split(":", 1) + config = {} + + for pair in config_str.split(","): + if "=" in pair: + key, value = pair.split("=", 1) + config[key] = value + + return name, config diff --git a/src/checkup/configuration/__init__.py b/src/checkup/configuration/__init__.py new file mode 100644 index 0000000..84328d0 --- /dev/null +++ b/src/checkup/configuration/__init__.py @@ -0,0 +1,18 @@ +"""CheckUp config files.""" + +from checkup.configuration.io import find_config_files, load_config +from checkup.configuration.models import ( + CheckupConfig, + MaterializerConfig, + MetricConfig, + ProviderConfig, +) + +__all__ = [ + "CheckupConfig", + "MaterializerConfig", + "MetricConfig", + "ProviderConfig", + "find_config_files", + "load_config", +] diff --git a/src/checkup/configuration/env.py b/src/checkup/configuration/env.py new file mode 100644 index 0000000..75d5e52 --- /dev/null +++ b/src/checkup/configuration/env.py @@ -0,0 +1,135 @@ +""" +Environment variable handling for configuration. +""" + +import logging +import os +import re +from typing import Any + +logger = logging.getLogger(__name__) + + +def substitute_env_vars(value: Any) -> Any: + """ + Recursively substitute ${VAR} patterns with environment variables. + + Supports: + - ${VAR} - substitute with env var value + - ${VAR:-default} - substitute with default if VAR not set + + Args: + value: Value to process (string, dict, list, or other) + + Returns: + Value with environment variables substituted + """ + + if isinstance(value, str): + pattern = r"\$\{([^}:]+)(?::-([^}]*))?\}" + + def replace(match: re.Match) -> str: + var_name = match.group(1) + default = match.group(2) + env_value = os.environ.get(var_name) + if env_value is not None: + return env_value + if default is not None: + return default + logger.warning("Environment variable %s not found", var_name) + return match.group(0) + + return re.sub(pattern, replace, value) + + elif isinstance(value, dict): + return {k: substitute_env_vars(v) for k, v in value.items()} + + elif isinstance(value, list): + return [substitute_env_vars(item) for item in value] + + return value + + +def apply_naming_convention_overrides(config: dict[str, Any]) -> dict[str, Any]: + """ + Apply CHECKUP__* environment variable overrides. + + Environment variables like CHECKUP__MATERIALIZER__SQLALCHEMY__CONNECTION_URL + override corresponding config values (only if not already set). + + Args: + config: Configuration dict to apply overrides to + + Returns: + Configuration with environment variable overrides applied + """ + + prefix = "CHECKUP__" + + for key, value in os.environ.items(): + if not key.startswith(prefix): + continue + + parts = key[len(prefix) :].lower().split("__") + if len(parts) < 2: + continue + + section = parts[0] + + if section == "materializer" and len(parts) >= 3: + _apply_materializer_override(config, parts, value, key) + + elif section == "provider" and len(parts) >= 3: + _apply_provider_override(config, parts, value, key) + + return config + + +def _apply_materializer_override( + config: dict[str, Any], + parts: list[str], + value: str, + key: str, +) -> None: + """ + Apply a materializer config override from env var. + """ + + materializer_type = parts[1] + config_key = "_".join(parts[2:]) + + if "materializer" not in config: + return + + mat_type = config.get("materializer", {}).get("type", "") + if mat_type and mat_type.lower() == materializer_type: + if config_key not in config["materializer"]: + config["materializer"][config_key] = value + logger.debug("Applied env override: %s", key) + + +def _apply_provider_override( + config: dict[str, Any], + parts: list[str], + value: str, + key: str, +) -> None: + """ + Apply a provider config override from env var. + """ + + provider_name = parts[1] + config_key = "_".join(parts[2:]) + + if "providers" not in config: + return + + for provider in config["providers"]: + if isinstance(provider, dict): + name = list(provider.keys())[0] if provider else None + if name and name.lower() == provider_name: + if provider[name] is None: + provider[name] = {} + if config_key not in provider.get(name, {}): + provider[name][config_key] = value + logger.debug("Applied env override: %s", key) diff --git a/src/checkup/configuration/io.py b/src/checkup/configuration/io.py new file mode 100644 index 0000000..527d725 --- /dev/null +++ b/src/checkup/configuration/io.py @@ -0,0 +1,212 @@ +""" +Configuration file I/O and merging. +""" + +import logging +from pathlib import Path +from typing import Any + +import yaml + +from checkup.configuration.env import ( + apply_naming_convention_overrides, + substitute_env_vars, +) +from checkup.configuration.models import ( + CheckupConfig, + MaterializerConfig, + MetricConfig, + ProviderConfig, +) + +logger = logging.getLogger(__name__) + +CONFIG_FILENAME = "checkup.yaml" + + +def load_yaml_file(path: Path) -> dict[str, Any]: + """ + Load a single YAML file. + """ + + if not path.exists(): + return {} + + try: + with open(path) as f: + data = yaml.safe_load(f) + return data if data else {} + except yaml.YAMLError as e: + logger.error("Failed to parse YAML config %s: %s", path, e) + raise + + +def find_config_files(start_dir: Path) -> list[Path]: + """ + Find all checkup.yaml files from start_dir up to filesystem root. + + Returns: + List of paths, ordered from root to start_dir (for merging) + """ + + config_files = [] + current = start_dir.resolve() + + while True: + config_path = current / CONFIG_FILENAME + if config_path.exists(): + config_files.append(config_path) + + parent = current.parent + if parent == current: + break + current = parent + + return list(reversed(config_files)) + + +def merge_configs(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + """ + Merge two configuration dicts (override wins). + """ + + result = base.copy() + + for key, value in override.items(): + if key == "tags" and key in result: + result[key] = {**result.get(key, {}), **value} + elif key == "providers": + result[key] = value + elif key == "metrics": + if key in result: + base_metrics = { + (m if isinstance(m, str) else list(m.keys())[0]): m + for m in result.get(key, []) + } + for metric in value: + name = metric if isinstance(metric, str) else list(metric.keys())[0] + base_metrics[name] = metric + result[key] = list(base_metrics.values()) + else: + result[key] = value + else: + result[key] = value + + return result + + +def parse_providers(raw: list[Any] | None) -> list[ProviderConfig]: + """ + Parse provider configuration from raw YAML. + + Supports: + - name: git + - name: dbt + project_dir: ./dbt + """ + + if not raw: + return [] + + providers = [] + for item in raw: + if isinstance(item, str): + providers.append(ProviderConfig(name=item)) + elif isinstance(item, dict): + name = item.get("name") + if name: + config = {k: v for k, v in item.items() if k != "name"} + providers.append(ProviderConfig(name=name, config=config)) + return providers + + +def parse_metrics(raw: list[Any] | None) -> list[MetricConfig]: + """ + Parse metric configuration from raw YAML. + + Supports: + - name: git_days_since_last_update + - name: python_version + version: "3.12" + """ + + if not raw: + return [] + + metrics = [] + for item in raw: + if isinstance(item, str): + metrics.append(MetricConfig(name=item)) + elif isinstance(item, dict): + name = item.get("name") + if name: + config = {k: v for k, v in item.items() if k != "name"} + metrics.append(MetricConfig(name=name, config=config)) + return metrics + + +def parse_materializer(raw: dict[str, Any] | None) -> MaterializerConfig | None: + """ + Parse materializer configuration from raw YAML. + """ + + if not raw: + return None + + raw_copy = raw.copy() + mat_type = raw_copy.pop("type", None) + if not mat_type: + return None + + return MaterializerConfig(type=mat_type, config=raw_copy) + + +def load_config( + config_path: Path | None = None, + start_dir: Path | None = None, +) -> CheckupConfig: + """ + Load checkup configuration with hierarchy and env var substitution. + + Resolution order: + 1. Find all checkup.yaml files from start_dir up to root + 2. Merge configs (child overrides parent) + 3. Apply naming convention env vars (CHECKUP__*) + 4. Substitute ${VAR} references + + Args: + config_path: Explicit config file path (skips hierarchy search) + start_dir: Directory to start searching from (defaults to cwd) + + Returns: + Merged and resolved CheckupConfig + """ + + if config_path: + raw = load_yaml_file(config_path) + else: + start = start_dir or Path.cwd() + config_files = find_config_files(start) + + if not config_files: + logger.debug("No config files found") + return CheckupConfig.empty() + + raw = {} + for cf in config_files: + logger.debug("Loading config: %s", cf) + file_config = load_yaml_file(cf) + raw = merge_configs(raw, file_config) + + # Apply naming convention env vars first (lowest priority) + raw = apply_naming_convention_overrides(raw) + + # Then substitute ${VAR} references (highest priority) + raw = substitute_env_vars(raw) + + return CheckupConfig( + tags=raw.get("tags", {}), + providers=parse_providers(raw.get("providers")), + metrics=parse_metrics(raw.get("metrics")), + materializer=parse_materializer(raw.get("materializer")), + ) diff --git a/src/checkup/configuration/models.py b/src/checkup/configuration/models.py new file mode 100644 index 0000000..5a499ea --- /dev/null +++ b/src/checkup/configuration/models.py @@ -0,0 +1,41 @@ +""" +Configuration models. +""" + +from typing import Any + +from pydantic import BaseModel, Field + + +class ProviderConfig(BaseModel): + """Configuration for a single provider.""" + + name: str + config: dict[str, Any] = Field(default_factory=dict) + + +class MetricConfig(BaseModel): + """Configuration for a single metric.""" + + name: str + config: dict[str, Any] = Field(default_factory=dict) + + +class MaterializerConfig(BaseModel): + """Configuration for the materializer.""" + + type: str + config: dict[str, Any] = Field(default_factory=dict) + + +class CheckupConfig(BaseModel): + """Complete checkup configuration.""" + + tags: dict[str, Any] = Field(default_factory=dict) + providers: list[ProviderConfig] = Field(default_factory=list) + metrics: list[MetricConfig] = Field(default_factory=list) + materializer: MaterializerConfig | None = None + + @classmethod + def empty(cls) -> "CheckupConfig": + return cls() diff --git a/src/checkup/configuration/schema.py b/src/checkup/configuration/schema.py new file mode 100644 index 0000000..062e3ed --- /dev/null +++ b/src/checkup/configuration/schema.py @@ -0,0 +1,246 @@ +""" +JSON Schema generation for checkup.yaml. +""" + +import json +from pathlib import Path +from typing import Any + +from checkup.registry import get_registry + +SCHEMA_VERSION = "https://json-schema.org/draft/2020-12/schema" +SCHEMA_ID = "https://checkup.dev/schemas/checkup.yaml.json" + + +def _get_pydantic_schema(cls: type) -> dict | None: + """ + Get JSON schema from a Pydantic model, filtering internal fields. + """ + + if not hasattr(cls, "model_json_schema"): + return None + + try: + schema = cls.model_json_schema() + + # Remove internal fields for metrics + if "properties" in schema: + for field in ("value", "diagnostic", "tags"): + schema["properties"].pop(field, None) + if "required" in schema: + schema["required"] = [ + r + for r in schema["required"] + if r not in ("value", "diagnostic", "tags") + ] + + # Remove $defs if present (inline everything) + schema.pop("$defs", None) + schema.pop("title", None) + + if not schema.get("properties"): + return None + + return schema + except Exception: + return None + + +def _get_provider_schema(cls: type) -> dict | None: + """ + Get JSON schema for a provider from its __init__ signature. + """ + + import inspect + from typing import get_type_hints + + try: + sig = inspect.signature(cls.__init__) + hints = get_type_hints(cls.__init__) + except Exception: + return None + + properties = {} + required = [] + + for name, param in sig.parameters.items(): + if name == "self": + continue + + prop: dict[str, Any] = {} + + # Get type + if name in hints: + hint = hints[name] + type_name = getattr(hint, "__name__", str(hint)) + if "str" in type_name or "str" in str(hint): + prop["type"] = "string" + elif "int" in type_name: + prop["type"] = "integer" + elif "float" in type_name: + prop["type"] = "number" + elif "bool" in type_name: + prop["type"] = "boolean" + elif "Path" in str(hint): + prop["type"] = "string" + else: + prop["type"] = "string" + else: + prop["type"] = "string" + + # Get default + if param.default is not inspect.Parameter.empty: + default = param.default + # Convert Path to string for JSON + if hasattr(default, "__fspath__"): + default = str(default) + prop["default"] = default + else: + required.append(name) + + properties[name] = prop + + if not properties: + return None + + schema: dict[str, Any] = { + "type": "object", + "properties": properties, + } + if required: + schema["required"] = required + + return schema + + +def generate_schema() -> dict: + """ + Generate JSON schema for checkup.yaml configuration. + + Dynamically includes available providers, metrics, and materializers + using Pydantic's schema generation. + """ + + registry = get_registry() + + # Collect provider info + provider_names = sorted(registry.providers.keys()) + provider_schemas = {} + for name, cls in registry.providers.items(): + schema = _get_provider_schema(cls) + if schema: + provider_schemas[name] = schema + + # Collect metric info + metric_names = sorted(registry.metrics.keys()) + metric_schemas = {} + for name, cls in registry.metrics.items(): + schema = _get_pydantic_schema(cls) + if schema: + metric_schemas[name] = schema + + # Collect materializer info + materializer_names = sorted(registry.materializers.keys()) + materializer_schemas = {} + for name, cls in registry.materializers.items(): + schema = _get_pydantic_schema(cls) + if schema: + materializer_schemas[name] = schema + + # Build provider item schema with oneOf for each provider + provider_variants = [] + for name in provider_names: + variant: dict[str, Any] = { + "type": "object", + "properties": { + "name": {"const": name}, + }, + "required": ["name"], + "additionalProperties": False, + } + if name in provider_schemas and "properties" in provider_schemas[name]: + variant["properties"].update(provider_schemas[name]["properties"]) + provider_variants.append(variant) + + provider_item_schema: dict[str, Any] = ( + {"oneOf": provider_variants} if provider_variants else {"type": "object"} + ) + + # Build metric item schema with oneOf for each metric + metric_variants = [] + for name in metric_names: + variant: dict[str, Any] = { + "type": "object", + "properties": { + "name": {"const": name}, + }, + "required": ["name"], + "additionalProperties": False, + } + if name in metric_schemas and "properties" in metric_schemas[name]: + variant["properties"].update(metric_schemas[name]["properties"]) + metric_variants.append(variant) + + metric_item_schema: dict[str, Any] = ( + {"oneOf": metric_variants} if metric_variants else {"type": "object"} + ) + + # Build materializer schema + materializer_props: dict[str, Any] = { + "type": { + "type": "string", + "enum": materializer_names, + } + if materializer_names + else {"type": "string"} + } + + # Add properties from all materializers + for schema in materializer_schemas.values(): + if "properties" in schema: + for prop_name, prop_schema in schema["properties"].items(): + if prop_name not in materializer_props: + materializer_props[prop_name] = prop_schema + + return { + "$schema": SCHEMA_VERSION, + "$id": SCHEMA_ID, + "title": "Checkup Configuration", + "description": "Configuration file for checkup", + "type": "object", + "properties": { + "tags": { + "type": "object", + "description": "Tags to identify the data product (e.g., product, team)", + "additionalProperties": {"type": "string"}, + }, + "providers": { + "type": "array", + "description": "Data providers for context enrichment", + "items": provider_item_schema, + }, + "metrics": { + "type": "array", + "description": "Metrics to calculate", + "items": metric_item_schema, + }, + "materializer": { + "type": "object", + "description": "Output materializer configuration", + "properties": materializer_props, + "required": ["type"], + }, + }, + "additionalProperties": False, + } + + +def write_schema(output_path: Path) -> None: + """ + Write JSON schema to file. + """ + + schema = generate_schema() + with open(output_path, "w") as f: + json.dump(schema, f, indent=2) + f.write("\n") diff --git a/src/checkup/registry/__init__.py b/src/checkup/registry/__init__.py new file mode 100644 index 0000000..7b3a1c4 --- /dev/null +++ b/src/checkup/registry/__init__.py @@ -0,0 +1,13 @@ +""" +Plugin registry for discovering providers, metrics, and materializers. +""" + +from checkup.registry.discovery import ( + PluginRegistry, + get_registry, +) + +__all__ = [ + "PluginRegistry", + "get_registry", +] diff --git a/src/checkup/registry/discovery.py b/src/checkup/registry/discovery.py new file mode 100644 index 0000000..9ac0627 --- /dev/null +++ b/src/checkup/registry/discovery.py @@ -0,0 +1,190 @@ +""" +Plugin discovery via Python entry points. +""" + +import logging +from importlib.metadata import entry_points +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from checkup.materializers import Materializer + from checkup.metric import Metric + from checkup.provider import Provider + +logger = logging.getLogger(__name__) + +# Entry point group names +PROVIDERS_GROUP = "checkup.providers" +METRICS_GROUP = "checkup.metrics" +MATERIALIZERS_GROUP = "checkup.materializers" + + +class PluginRegistry: + """ + Registry for discovering and loading checkup plugins. + + Plugins register providers, metrics, and materializers via entry points + in their pyproject.toml. + """ + + def __init__(self) -> None: + """ + Initialize the registry. + """ + + self._providers: dict[str, type[Provider]] | None = None + self._metrics: dict[str, type[Metric]] | None = None + self._materializers: dict[str, type[Materializer]] | None = None + + def _list_entry_point_names(self, group: str) -> list[str]: + """ + List entry point names without loading them. + """ + + eps = entry_points(group=group) + return [ep.name for ep in eps] + + def _load_entry_points(self, group: str) -> dict[str, type]: + """ + Load all entry points for a group. + """ + + result = {} + eps = entry_points(group=group) + + for ep in eps: + try: + cls = ep.load() + result[ep.name] = cls + logger.debug("Loaded %s: %s", group, ep.name) + except Exception as e: + logger.warning("Failed to load %s '%s': %s", group, ep.name, e) + + return result + + @property + def providers(self) -> dict[str, type["Provider"]]: + """ + Get all registered providers. + """ + + if self._providers is None: + self._providers = self._load_entry_points(PROVIDERS_GROUP) + + return self._providers + + @property + def metrics(self) -> dict[str, type["Metric"]]: + """ + Get all registered metrics. + """ + + if self._metrics is None: + self._metrics = self._load_entry_points(METRICS_GROUP) + + return self._metrics + + @property + def materializers(self) -> dict[str, type["Materializer"]]: + """ + Get all registered materializers. + """ + + if self._materializers is None: + self._materializers = self._load_entry_points(MATERIALIZERS_GROUP) + + return self._materializers + + def get_provider(self, name: str) -> type["Provider"] | None: + """ + Get a provider class by name. + """ + + return self.providers.get(name) + + def get_metric(self, name: str) -> type["Metric"] | None: + """ + Get a metric class by name. + """ + + return self.metrics.get(name) + + def get_materializer(self, name: str) -> type["Materializer"] | None: + """ + Get a materializer class by name. + """ + + return self.materializers.get(name) + + def list_provider_names(self) -> list[str]: + """ + List available provider names without loading them. + """ + + return self._list_entry_point_names(PROVIDERS_GROUP) + + def list_metric_names(self) -> list[str]: + """ + List available metric names without loading them. + """ + + return self._list_entry_point_names(METRICS_GROUP) + + def list_materializer_names(self) -> list[str]: + """ + List available materializer names without loading them. + """ + + return self._list_entry_point_names(MATERIALIZERS_GROUP) + + def list_compatible_metric_names(self, provider_names: list[str]) -> list[str]: + """ + List metric names compatible with the given providers. + + A metric is compatible if all its required providers (from providers() method) + are in the selected provider list. Metrics with no required providers are + always compatible. + """ + + provider_set = set(provider_names) + compatible = [] + + for name, metric_cls in self.metrics.items(): + required_providers = metric_cls.providers() + + if not required_providers: + # No provider requirements - always compatible + compatible.append(name) + continue + + # Check if all required providers are selected + required_names = {p.name for p in required_providers} + if required_names <= provider_set: + compatible.append(name) + + return compatible + + def clear_cache(self) -> None: + """ + Clear cached plugins (useful for testing). + """ + + self._providers = None + self._metrics = None + self._materializers = None + + +# Global registry instance +_registry: PluginRegistry | None = None + + +def get_registry() -> PluginRegistry: + """ + Get the global plugin registry. + """ + + global _registry + if _registry is None: + _registry = PluginRegistry() + + return _registry diff --git a/tests/test_cli_configuration.py b/tests/test_cli_configuration.py new file mode 100644 index 0000000..8a7a4c6 --- /dev/null +++ b/tests/test_cli_configuration.py @@ -0,0 +1,288 @@ +""" +Tests for CLI configuration loading and parsing. +""" + +from typing import ClassVar + +import yaml + +from checkup.configuration.env import ( + apply_naming_convention_overrides, + substitute_env_vars, +) +from checkup.configuration.io import ( + load_config, + merge_configs, + parse_materializer, + parse_metrics, + parse_providers, +) +from checkup.configuration.models import CheckupConfig +from checkup.metric import Metric +from checkup.types import Context + + +class TestParseProviders: + def test_string_shorthand_creates_provider_with_empty_config(self): + raw = ["git", "dbt"] + result = parse_providers(raw) + + assert len(result) == 2 + assert result[0].name == "git" + assert result[0].config == {} + assert result[1].name == "dbt" + + def test_dict_with_name_field_extracts_config(self): + raw = [ + {"name": "git", "repo_path": "/path/to/repo"}, + {"name": "dbt", "project_dir": "./dbt", "profiles_dir": "~/.dbt"}, + ] + result = parse_providers(raw) + + assert len(result) == 2 + assert result[0].name == "git" + assert result[0].config == {"repo_path": "/path/to/repo"} + assert result[1].name == "dbt" + assert result[1].config == {"project_dir": "./dbt", "profiles_dir": "~/.dbt"} + + def test_mixed_string_and_dict_formats(self): + raw = [ + "git", + {"name": "dbt", "project_dir": "./dbt"}, + ] + result = parse_providers(raw) + + assert len(result) == 2 + assert result[0].name == "git" + assert result[0].config == {} + assert result[1].name == "dbt" + assert result[1].config == {"project_dir": "./dbt"} + + def test_empty_list_returns_empty(self): + assert parse_providers([]) == [] + assert parse_providers(None) == [] + + +class TestParseMetrics: + def test_string_shorthand_creates_metric_with_empty_config(self): + raw = ["git_days_since_last_update", "python_version"] + result = parse_metrics(raw) + + assert len(result) == 2 + assert result[0].name == "git_days_since_last_update" + assert result[0].config == {} + + def test_dict_with_name_field_extracts_config(self): + raw = [ + { + "name": "python_version_check", + "min_version": "3.10", + "max_version": "3.13", + }, + ] + result = parse_metrics(raw) + + assert len(result) == 1 + assert result[0].name == "python_version_check" + assert result[0].config == {"min_version": "3.10", "max_version": "3.13"} + + +class TestParseMaterializer: + def test_extracts_type_and_remaining_fields_as_config(self): + raw = {"type": "console", "group_tag_1": "product", "group_tag_2": "team"} + result = parse_materializer(raw) + + assert result.type == "console" + assert result.config == {"group_tag_1": "product", "group_tag_2": "team"} + + def test_returns_none_when_type_missing(self): + assert parse_materializer({"group_tag_1": "product"}) is None + assert parse_materializer(None) is None + + +class TestMergeConfigs: + def test_child_tags_merged_with_parent(self): + parent = {"tags": {"team": "platform", "env": "prod"}} + child = {"tags": {"product": "my-product"}} + + result = merge_configs(parent, child) + + assert result["tags"] == { + "team": "platform", + "env": "prod", + "product": "my-product", + } + + def test_child_tag_overrides_parent(self): + parent = {"tags": {"env": "prod"}} + child = {"tags": {"env": "dev"}} + + result = merge_configs(parent, child) + + assert result["tags"]["env"] == "dev" + + def test_child_providers_replace_parent(self): + parent = {"providers": [{"name": "git"}]} + child = {"providers": [{"name": "dbt"}]} + + result = merge_configs(parent, child) + + assert result["providers"] == [{"name": "dbt"}] + + def test_child_materializer_replaces_parent(self): + parent = {"materializer": {"type": "csv"}} + child = {"materializer": {"type": "console"}} + + result = merge_configs(parent, child) + + assert result["materializer"]["type"] == "console" + + +class TestSubstituteEnvVars: + def test_substitutes_env_var_reference(self, monkeypatch): + monkeypatch.setenv("MY_VAR", "secret_value") + config = {"password": "${MY_VAR}"} + + result = substitute_env_vars(config) + + assert result["password"] == "secret_value" + + def test_uses_default_when_var_not_set(self): + config = {"timeout": "${MISSING_VAR:-30}"} + + result = substitute_env_vars(config) + + assert result["timeout"] == "30" + + def test_leaves_unset_var_without_default_unchanged(self): + """Unset vars without defaults are left as-is (with warning logged).""" + config = {"value": "${DEFINITELY_NOT_SET}"} + + result = substitute_env_vars(config) + + assert result["value"] == "${DEFINITELY_NOT_SET}" + + def test_substitutes_in_nested_structures(self, monkeypatch): + monkeypatch.setenv("DB_HOST", "localhost") + config = { + "providers": [ + {"name": "db", "host": "${DB_HOST}"}, + ] + } + + result = substitute_env_vars(config) + + assert result["providers"][0]["host"] == "localhost" + + +class TestNamingConventionOverrides: + def test_materializer_override_applied_when_type_matches(self, monkeypatch): + monkeypatch.setenv( + "CHECKUP__MATERIALIZER__SQLALCHEMY__CONNECTION_URL", + "postgresql://localhost", + ) + config = {"materializer": {"type": "sqlalchemy"}} + + result = apply_naming_convention_overrides(config) + + assert result["materializer"]["connection_url"] == "postgresql://localhost" + + def test_materializer_override_skipped_when_type_differs(self, monkeypatch): + monkeypatch.setenv( + "CHECKUP__MATERIALIZER__SQLALCHEMY__CONNECTION_URL", + "postgresql://localhost", + ) + config = {"materializer": {"type": "console"}} + + result = apply_naming_convention_overrides(config) + + assert "connection_url" not in result["materializer"] + + def test_explicit_config_wins_over_naming_convention(self, monkeypatch): + monkeypatch.setenv( + "CHECKUP__MATERIALIZER__SQLALCHEMY__CONNECTION_URL", "env-url" + ) + config = {"materializer": {"type": "sqlalchemy", "connection_url": "yaml-url"}} + + result = apply_naming_convention_overrides(config) + + assert result["materializer"]["connection_url"] == "yaml-url" + + +class TestLoadConfig: + def test_loads_yaml_file(self, tmp_path): + config_file = tmp_path / "checkup.yaml" + config_file.write_text( + yaml.dump( + { + "tags": {"product": "test"}, + "providers": [{"name": "git"}], + "metrics": [{"name": "dummy_metric"}], + } + ) + ) + + result = load_config(config_path=config_file) + + assert isinstance(result, CheckupConfig) + assert result.tags == {"product": "test"} + assert len(result.providers) == 1 + assert result.providers[0].name == "git" + + def test_returns_empty_config_when_no_file_found(self, tmp_path): + result = load_config(start_dir=tmp_path) + + assert result.tags == {} + assert result.providers == [] + assert result.metrics == [] + + def test_hierarchical_loading_merges_parent_and_child(self, tmp_path): + # Create parent directory with config + parent_dir = tmp_path / "parent" + parent_dir.mkdir() + (parent_dir / "checkup.yaml").write_text( + yaml.dump({"tags": {"team": "platform"}}) + ) + + # Create child directory with config + child_dir = parent_dir / "child" + child_dir.mkdir() + (child_dir / "checkup.yaml").write_text( + yaml.dump({"tags": {"product": "my-product"}}) + ) + + result = load_config(start_dir=child_dir) + + assert result.tags == {"team": "platform", "product": "my-product"} + + +class ConfigurableMetric(Metric): + """Test metric with required config fields.""" + + name: ClassVar[str] = "configurable" + description: ClassVar[str] = "Needs config" + unit: ClassVar[str] = "count" + + multiplier: int + offset: int = 0 + + def calculate(self, _context: Context, _metrics: dict) -> None: + self.value = 10 * self.multiplier + self.offset + + +class TestHubMetricConfigs: + def test_configured_subclass_passed_to_hub(self): + from checkup.hub import CheckHub + from checkup.metric import create_configured_metric + + # Create a configured subclass (this is how CLI executor does it) + Configured = create_configured_metric( + ConfigurableMetric, + {"multiplier": 5, "offset": 3}, + ) + + hub = CheckHub().with_metrics([Configured]) + result = hub.measure() + + assert len(result.metrics) == 1 + assert result.metrics[0].value == 53 # 10 * 5 + 3 diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..b57cb96 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,187 @@ +""" +Tests for plugin registry and discovery. +""" + +from typing import ClassVar +from unittest.mock import MagicMock, patch + +from checkup.metric import Metric +from checkup.provider import Provider +from checkup.registry.discovery import PluginRegistry +from checkup.types import Context + + +class MockGitProvider(Provider): + name: ClassVar[str] = "git" + + def provide(self) -> dict: + return {} + + +class MockDbtProvider(Provider): + name: ClassVar[str] = "dbt" + + def provide(self) -> dict: + return {} + + +class MetricRequiringGit(Metric): + name: ClassVar[str] = "git_metric" + description: ClassVar[str] = "Requires git" + unit: ClassVar[str] = "count" + + @classmethod + def providers(cls) -> list[type[Provider]]: + return [MockGitProvider] + + def calculate(self, _context: Context, _metrics: dict) -> None: + self.value = 1 + + +class MetricRequiringDbt(Metric): + name: ClassVar[str] = "dbt_metric" + description: ClassVar[str] = "Requires dbt" + unit: ClassVar[str] = "count" + + @classmethod + def providers(cls) -> list[type[Provider]]: + return [MockDbtProvider] + + def calculate(self, _context: Context, _metrics: dict) -> None: + self.value = 1 + + +class MetricRequiringBoth(Metric): + name: ClassVar[str] = "both_metric" + description: ClassVar[str] = "Requires git and dbt" + unit: ClassVar[str] = "count" + + @classmethod + def providers(cls) -> list[type[Provider]]: + return [MockGitProvider, MockDbtProvider] + + def calculate(self, _context: Context, _metrics: dict) -> None: + self.value = 1 + + +class StandaloneMetric(Metric): + name: ClassVar[str] = "standalone" + description: ClassVar[str] = "No providers required" + unit: ClassVar[str] = "count" + + def calculate(self, _context: Context, _metrics: dict) -> None: + self.value = 1 + + +class TestCompatibleMetricFiltering: + def test_metric_included_when_required_provider_selected(self): + registry = PluginRegistry() + registry._metrics = { + "git_metric": MetricRequiringGit, + "dbt_metric": MetricRequiringDbt, + } + + compatible = registry.list_compatible_metric_names(["git"]) + + assert "git_metric" in compatible + assert "dbt_metric" not in compatible + + def test_metric_excluded_when_required_provider_missing(self): + registry = PluginRegistry() + registry._metrics = { + "git_metric": MetricRequiringGit, + } + + compatible = registry.list_compatible_metric_names(["dbt"]) + + assert "git_metric" not in compatible + + def test_metric_requiring_multiple_providers_needs_all(self): + registry = PluginRegistry() + registry._metrics = { + "both_metric": MetricRequiringBoth, + } + + # Only git selected - not enough + compatible = registry.list_compatible_metric_names(["git"]) + assert "both_metric" not in compatible + + # Both selected - now compatible + compatible = registry.list_compatible_metric_names(["git", "dbt"]) + assert "both_metric" in compatible + + def test_standalone_metrics_always_compatible(self): + registry = PluginRegistry() + registry._metrics = { + "standalone": StandaloneMetric, + } + + # No providers selected + compatible = registry.list_compatible_metric_names([]) + assert "standalone" in compatible + + # Some providers selected + compatible = registry.list_compatible_metric_names(["git"]) + assert "standalone" in compatible + + +class TestRegistryListing: + def test_list_provider_names_without_loading(self): + """Listing names should not import any modules.""" + registry = PluginRegistry() + + with patch("checkup.registry.discovery.entry_points") as mock_eps: + mock_ep = MagicMock() + mock_ep.name = "git" + mock_eps.return_value = [mock_ep] + + names = registry.list_provider_names() + + assert names == ["git"] + mock_ep.load.assert_not_called() + + def test_list_metric_names_without_loading(self): + registry = PluginRegistry() + + with patch("checkup.registry.discovery.entry_points") as mock_eps: + mock_ep = MagicMock() + mock_ep.name = "git_days_since_last_update" + mock_eps.return_value = [mock_ep] + + names = registry.list_metric_names() + + assert names == ["git_days_since_last_update"] + mock_ep.load.assert_not_called() + + +class TestRegistryLoading: + def test_providers_property_loads_and_caches_entry_points(self): + registry = PluginRegistry() + + with patch("checkup.registry.discovery.entry_points") as mock_eps: + mock_ep = MagicMock() + mock_ep.name = "test_provider" + mock_ep.load.return_value = "LoadedProviderClass" + mock_eps.return_value = [mock_ep] + + providers1 = registry.providers + providers2 = registry.providers + + assert providers1 == {"test_provider": "LoadedProviderClass"} + assert providers1 is providers2 + assert mock_eps.call_count == 1 + + def test_clear_cache_allows_reload(self): + registry = PluginRegistry() + + with patch("checkup.registry.discovery.entry_points") as mock_eps: + mock_ep = MagicMock() + mock_ep.name = "test" + mock_ep.load.return_value = "Class" + mock_eps.return_value = [mock_ep] + + _ = registry.providers + registry.clear_cache() + _ = registry.providers + + assert mock_eps.call_count == 2 From a96bae713d8daa70072040845c50f1a7b13b52b3 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Mon, 20 Apr 2026 14:28:47 +0200 Subject: [PATCH 08/21] update tests --- plugins/checkup-dbt/tests/test_all_metrics.py | 40 +-- .../checkup-dbt/tests/test_core_metrics.py | 56 ++-- .../checkup-dbt/tests/test_output_metrics.py | 40 +-- plugins/checkup-dbt/tests/test_provider.py | 20 +- .../checkup-dbt/tests/test_quality_metrics.py | 82 ++--- .../checkup-dbt/tests/test_test_metrics.py | 44 +-- plugins/checkup-git/tests/test_git_metrics.py | 46 +-- .../src/checkup_python/metrics/version.py | 31 -- .../checkup-python/tests/test_python_hub.py | 10 +- .../tests/test_python_metric.py | 17 +- src/checkup/materializers/console.py | 30 +- tests/test_materializers.py | 312 ++++++++++-------- 12 files changed, 356 insertions(+), 372 deletions(-) diff --git a/plugins/checkup-dbt/tests/test_all_metrics.py b/plugins/checkup-dbt/tests/test_all_metrics.py index 7c2db50..0da3348 100644 --- a/plugins/checkup-dbt/tests/test_all_metrics.py +++ b/plugins/checkup-dbt/tests/test_all_metrics.py @@ -31,23 +31,23 @@ class Dbt19SupportedVersionMetric(DbtSupportedVersionMetric): def test_all_metrics(sample_manifest_path: Path): all_metrics = [ - DbtModelsMetric, - DbtColumnsMetric, - DbtTestsMetric, - DbtModelsWithDescriptionMetric, - DbtColumnsWithDescriptionMetric, - DbtUnitTestsMetric, - DbtDataTestsMetric, - DbtColumnTestsMetric, - DbtTestedColumnsMetric, - DbtColumnTestCoverageMetric, - DbtOutputModelsMetric, - DbtOutputModelsWithDescriptionMetric, - DbtOutputModelsWithoutContractsMetric, - DbtOutputColumnsWithoutDataTypeMetric, - InternalModelNamingMetric, - Dbt19SupportedVersionMetric, - DbtVersionMetric, + DbtModelsMetric(), + DbtColumnsMetric(), + DbtTestsMetric(), + DbtModelsWithDescriptionMetric(), + DbtColumnsWithDescriptionMetric(), + DbtUnitTestsMetric(), + DbtDataTestsMetric(), + DbtColumnTestsMetric(), + DbtTestedColumnsMetric(), + DbtColumnTestCoverageMetric(), + DbtOutputModelsMetric(), + DbtOutputModelsWithDescriptionMetric(), + DbtOutputModelsWithoutContractsMetric(), + DbtOutputColumnsWithoutDataTypeMetric(), + InternalModelNamingMetric(), + Dbt19SupportedVersionMetric(), + DbtVersionMetric(), ] result = ( @@ -57,8 +57,8 @@ def test_all_metrics(sample_manifest_path: Path): .measure() ) - assert len(result.metrics) == 17 + assert len(result.measurements) == 17 assert len(result.errors) == 0 - for metric in result.metrics: - assert metric.value is not None, f"{metric.name} has no value" + for measurement in result.measurements: + assert measurement.value is not None, f"{measurement.metric.name} has no value" diff --git a/plugins/checkup-dbt/tests/test_core_metrics.py b/plugins/checkup-dbt/tests/test_core_metrics.py index a0ae6d3..8bc97ff 100644 --- a/plugins/checkup-dbt/tests/test_core_metrics.py +++ b/plugins/checkup-dbt/tests/test_core_metrics.py @@ -17,89 +17,89 @@ def test_models_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtModelsMetric]) + .with_metrics([DbtModelsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_models" - assert metric.value == 3 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_models" + assert measurement.value == 3 def test_columns_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtColumnsMetric]) + .with_metrics([DbtColumnsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_columns" - assert metric.value == 12 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_columns" + assert measurement.value == 12 def test_tests_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtTestsMetric]) + .with_metrics([DbtTestsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_tests" - assert metric.value == 9 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_tests" + assert measurement.value == 9 def test_models_with_description_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtModelsWithDescriptionMetric]) + .with_metrics([DbtModelsWithDescriptionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_models_with_description" - assert metric.value == 3 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_models_with_description" + assert measurement.value == 3 def test_columns_with_description_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtColumnsWithDescriptionMetric]) + .with_metrics([DbtColumnsWithDescriptionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_columns_with_description" - assert metric.value == 10 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_columns_with_description" + assert measurement.value == 10 def test_models_without_description_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtModelsWithoutDescriptionMetric]) + .with_metrics([DbtModelsWithoutDescriptionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_models_without_description" - assert metric.value == 0 # All 3 models have descriptions + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_models_without_description" + assert measurement.value == 0 # All 3 models have descriptions def test_columns_without_description_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtColumnsWithoutDescriptionMetric]) + .with_metrics([DbtColumnsWithoutDescriptionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_columns_without_description" - assert metric.value == 2 # 12 total - 10 with descriptions = 2 without + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_columns_without_description" + assert measurement.value == 2 # 12 total - 10 with descriptions = 2 without diff --git a/plugins/checkup-dbt/tests/test_output_metrics.py b/plugins/checkup-dbt/tests/test_output_metrics.py index b6403af..8056382 100644 --- a/plugins/checkup-dbt/tests/test_output_metrics.py +++ b/plugins/checkup-dbt/tests/test_output_metrics.py @@ -15,63 +15,63 @@ def test_output_models_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtOutputModelsMetric]) + .with_metrics([DbtOutputModelsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_output_models" - assert metric.value == 1 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_output_models" + assert measurement.value == 1 def test_output_models_with_description_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtOutputModelsWithDescriptionMetric]) + .with_metrics([DbtOutputModelsWithDescriptionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_output_models_with_description" - assert metric.value == 1 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_output_models_with_description" + assert measurement.value == 1 def test_output_models_without_description_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtOutputModelsWithoutDescriptionMetric]) + .with_metrics([DbtOutputModelsWithoutDescriptionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_output_models_without_description" - assert metric.value == 0 # The 1 output model has a description + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_output_models_without_description" + assert measurement.value == 0 # The 1 output model has a description def test_output_models_without_contracts_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtOutputModelsWithoutContractsMetric]) + .with_metrics([DbtOutputModelsWithoutContractsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_output_models_without_contracts" - assert metric.value == 0 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_output_models_without_contracts" + assert measurement.value == 0 def test_output_columns_without_data_type_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtOutputColumnsWithoutDataTypeMetric]) + .with_metrics([DbtOutputColumnsWithoutDataTypeMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_output_columns_without_data_type" - assert metric.value == 0 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_output_columns_without_data_type" + assert measurement.value == 0 diff --git a/plugins/checkup-dbt/tests/test_provider.py b/plugins/checkup-dbt/tests/test_provider.py index d422580..b3b70e7 100644 --- a/plugins/checkup-dbt/tests/test_provider.py +++ b/plugins/checkup-dbt/tests/test_provider.py @@ -11,19 +11,19 @@ def test_manifest_path_mode(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtModelsMetric]) + .with_metrics([DbtModelsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - assert len(result.metrics) == 1 - assert result.metrics[0].value == 3 + assert len(result.measurements) == 1 + assert result.measurements[0].value == 3 def test_dbt_project_dir_mode(sample_dbt_project_dir: Path): result = ( CheckHub() - .with_metrics([DbtModelsMetric]) + .with_metrics([DbtModelsMetric()]) .with_providers( [ [ @@ -37,8 +37,8 @@ def test_dbt_project_dir_mode(sample_dbt_project_dir: Path): .measure() ) - assert len(result.metrics) == 1 - assert result.metrics[0].value == 3 + assert len(result.measurements) == 1 + assert result.measurements[0].value == 3 def test_missing_args_raises_error(): @@ -53,7 +53,7 @@ def test_missing_args_raises_error(): def test_multiple_projects(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtModelsMetric]) + .with_metrics([DbtModelsMetric()]) .with_providers( [ [ @@ -69,7 +69,7 @@ def test_multiple_projects(sample_manifest_path: Path): .measure() ) - assert len(result.metrics) == 2 - assert all(m.value == 3 for m in result.metrics) - projects = {m.tags["project"] for m in result.metrics} + assert len(result.measurements) == 2 + assert all(m.value == 3 for m in result.measurements) + projects = {m.tags["project"] for m in result.measurements} assert projects == {"project_a", "project_b"} diff --git a/plugins/checkup-dbt/tests/test_quality_metrics.py b/plugins/checkup-dbt/tests/test_quality_metrics.py index 7e66783..ea70ecb 100644 --- a/plugins/checkup-dbt/tests/test_quality_metrics.py +++ b/plugins/checkup-dbt/tests/test_quality_metrics.py @@ -16,29 +16,29 @@ def test_naming_convention_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([InternalModelNamingMetric]) + .with_metrics([InternalModelNamingMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_models_not_adhering_to_naming_convention" - assert metric.value == 0 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_models_not_adhering_to_naming_convention" + assert measurement.value == 0 def test_naming_convention_metric_custom_checker(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([FactDimNamingMetric]) + .with_metrics([FactDimNamingMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) assert len(result.errors) == 0, f"Errors: {result.errors}" - assert len(result.metrics) == 1 + assert len(result.measurements) == 1 - metric = result.metrics[0] - assert metric.value == 3 + measurement = result.measurements[0] + assert measurement.value == 3 class Dbt19SupportedVersionMetric(DbtSupportedVersionMetric): @@ -48,14 +48,16 @@ class Dbt19SupportedVersionMetric(DbtSupportedVersionMetric): def test_supported_version_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([Dbt19SupportedVersionMetric]) + .with_metrics([Dbt19SupportedVersionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = next(m for m in result.metrics if m.name == "dbt_supported_version") - assert metric.unit == "boolean" - assert metric.value == 1 + measurement = next( + m for m in result.measurements if m.metric.name == "dbt_supported_version" + ) + assert measurement.metric.unit == "boolean" + assert measurement.value == 1 def test_supported_version_metric_requires_min_version(): @@ -70,16 +72,16 @@ def test_supported_version_metric_requires_min_version(): def test_version_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtVersionMetric]) + .with_metrics([DbtVersionMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_version" - assert metric.unit == "version" - assert metric.value is not None - assert isinstance(metric.value, str) + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_version" + assert measurement.metric.unit == "version" + assert measurement.value is not None + assert isinstance(measurement.value, str) class FlaggedPackageMetric(DbtFlaggedPackagesMetric): @@ -89,7 +91,7 @@ class FlaggedPackageMetric(DbtFlaggedPackagesMetric): def test_flagged_packages_metric(sample_manifest_path_with_git_packages: Path): result = ( CheckHub() - .with_metrics([FlaggedPackageMetric]) + .with_metrics([FlaggedPackageMetric()]) .with_providers( [ [ @@ -102,11 +104,11 @@ def test_flagged_packages_metric(sample_manifest_path_with_git_packages: Path): .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_flagged_packages" - assert metric.unit == "packages" - assert metric.value == 1 - assert "flagged-package" in metric.diagnostic + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_flagged_packages" + assert measurement.metric.unit == "packages" + assert measurement.value == 1 + assert "flagged-package" in measurement.diagnostic class NoFlaggedPackageMetric(DbtFlaggedPackagesMetric): @@ -118,7 +120,7 @@ def test_flagged_packages_metric_no_matches( ): result = ( CheckHub() - .with_metrics([NoFlaggedPackageMetric]) + .with_metrics([NoFlaggedPackageMetric()]) .with_providers( [ [ @@ -131,9 +133,9 @@ def test_flagged_packages_metric_no_matches( .measure() ) - metric = result.metrics[0] - assert metric.value == 0 - assert not metric.diagnostic + measurement = result.measurements[0] + assert measurement.value == 0 + assert not measurement.diagnostic def test_flagged_packages_metric_requires_flagged_packages(): @@ -152,17 +154,17 @@ class DevProfileHostMetric(DbtProfileHostMetric): def test_profile_host_metric_dev(sample_manifest_path_with_host: Path): result = ( CheckHub() - .with_metrics([DevProfileHostMetric]) + .with_metrics([DevProfileHostMetric()]) .with_providers( [[DbtManifestProvider(manifest_path=sample_manifest_path_with_host)]] ) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_profile_host" - assert metric.unit == "url" - assert metric.value == "myprefix.minerva.dp-ond.vlaanderen.be" + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_profile_host" + assert measurement.metric.unit == "url" + assert measurement.value == "myprefix.minerva.dp-ond.vlaanderen.be" class ProdProfileHostMetric(DbtProfileHostMetric): @@ -172,29 +174,29 @@ class ProdProfileHostMetric(DbtProfileHostMetric): def test_profile_host_metric_prod(sample_manifest_path_with_host: Path): result = ( CheckHub() - .with_metrics([ProdProfileHostMetric]) + .with_metrics([ProdProfileHostMetric()]) .with_providers( [[DbtManifestProvider(manifest_path=sample_manifest_path_with_host)]] ) .measure() ) - metric = result.metrics[0] - assert metric.value == "prod.example.com" + measurement = result.measurements[0] + assert measurement.value == "prod.example.com" def test_profile_host_metric_no_host(sample_manifest_path: Path): """Test when profiles.yml has no host configured.""" result = ( CheckHub() - .with_metrics([DevProfileHostMetric]) + .with_metrics([DevProfileHostMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.value is None - assert "No host found" in metric.diagnostic + measurement = result.measurements[0] + assert measurement.value is None + assert "No host found" in measurement.diagnostic def test_profile_host_metric_requires_target(): diff --git a/plugins/checkup-dbt/tests/test_test_metrics.py b/plugins/checkup-dbt/tests/test_test_metrics.py index eafe056..68eada2 100644 --- a/plugins/checkup-dbt/tests/test_test_metrics.py +++ b/plugins/checkup-dbt/tests/test_test_metrics.py @@ -15,67 +15,67 @@ def test_unit_tests_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtUnitTestsMetric]) + .with_metrics([DbtUnitTestsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_unit_tests" - assert metric.value == 1 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_unit_tests" + assert measurement.value == 1 def test_data_tests_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtDataTestsMetric]) + .with_metrics([DbtDataTestsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_data_tests" - assert metric.value == 8 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_data_tests" + assert measurement.value == 8 def test_column_tests_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtColumnTestsMetric]) + .with_metrics([DbtColumnTestsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_column_tests" - assert metric.value == 8 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_column_tests" + assert measurement.value == 8 def test_tested_columns_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtTestedColumnsMetric]) + .with_metrics([DbtTestedColumnsMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dbt_tested_columns" - assert metric.value == 5 + measurement = result.measurements[0] + assert measurement.metric.name == "dbt_tested_columns" + assert measurement.value == 5 def test_column_test_coverage_metric(sample_manifest_path: Path): result = ( CheckHub() - .with_metrics([DbtColumnTestCoverageMetric]) + .with_metrics([DbtColumnTestCoverageMetric()]) .with_providers([[DbtManifestProvider(manifest_path=sample_manifest_path)]]) .measure() ) - assert len(result.metrics) == 3 + assert len(result.measurements) == 3 - coverage_metric = next( - m for m in result.metrics if m.name == "dbt_column_test_coverage" + coverage_measurement = next( + m for m in result.measurements if m.metric.name == "dbt_column_test_coverage" ) - assert coverage_metric.unit == "percent" - assert coverage_metric.value == 41 + assert coverage_measurement.metric.unit == "percent" + assert coverage_measurement.value == 41 diff --git a/plugins/checkup-git/tests/test_git_metrics.py b/plugins/checkup-git/tests/test_git_metrics.py index 2880d7f..b9f383d 100644 --- a/plugins/checkup-git/tests/test_git_metrics.py +++ b/plugins/checkup-git/tests/test_git_metrics.py @@ -13,28 +13,28 @@ def test_days_since_last_update_metric(git_repo: Path): """Test days since last update metric.""" result = ( CheckHub() - .with_metrics([GitDaysSinceLastUpdateMetric]) + .with_metrics([GitDaysSinceLastUpdateMetric()]) .with_providers([[GitProvider(repo_path=git_repo)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "git_days_since_last_update" - assert metric.value == 0 # Just committed, should be 0 days + measurement = result.measurements[0] + assert measurement.metric.name == "git_days_since_last_update" + assert measurement.value == 0 # Just committed, should be 0 days def test_tracked_file_count_metric(git_repo: Path): """Test tracked file count metric.""" result = ( CheckHub() - .with_metrics([GitTrackedFileCountMetric]) + .with_metrics([GitTrackedFileCountMetric()]) .with_providers([[GitProvider(repo_path=git_repo)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "git_tracked_file_count" - assert metric.value == 1 # One file (README.md) in the fixture + measurement = result.measurements[0] + assert measurement.metric.name == "git_tracked_file_count" + assert measurement.value == 1 # One file (README.md) in the fixture class DagCountMetric(GitTrackedFileCountMetric): @@ -85,14 +85,14 @@ def test_tracked_file_count_with_pattern(tmp_path: Path): # Test filtering by directory and pattern result = ( CheckHub() - .with_metrics([DagCountMetric]) + .with_metrics([DagCountMetric()]) .with_providers([[GitProvider(repo_path=repo_path)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "dag_count" - assert metric.value == 2 # Only dag1.py and dag2.py + measurement = result.measurements[0] + assert measurement.metric.name == "dag_count" + assert measurement.value == 2 # Only dag1.py and dag2.py class ReadmeExistsMetric(GitTrackedFileCountMetric): @@ -105,14 +105,14 @@ def test_file_exists_metric_when_file_exists(git_repo: Path): """Test file exists metric returns True when file exists.""" result = ( CheckHub() - .with_metrics([ReadmeExistsMetric]) + .with_metrics([ReadmeExistsMetric()]) .with_providers([[GitProvider(repo_path=git_repo)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "readme_exists" - assert metric.value == 1 + measurement = result.measurements[0] + assert measurement.metric.name == "readme_exists" + assert measurement.value == 1 class CruftFileExistsMetric(GitTrackedFileCountMetric): @@ -125,14 +125,14 @@ def test_tracked_file_count_when_no_match(git_repo: Path): """Test tracked file count returns 0 when pattern doesn't match.""" result = ( CheckHub() - .with_metrics([CruftFileExistsMetric]) + .with_metrics([CruftFileExistsMetric()]) .with_providers([[GitProvider(repo_path=git_repo)]]) .measure() ) - metric = result.metrics[0] - assert metric.name == "cruft_file_exists" - assert metric.value == 0 + measurement = result.measurements[0] + assert measurement.metric.name == "cruft_file_exists" + assert measurement.value == 0 def test_provider_returns_last_commit_date(git_repo: Path): @@ -236,10 +236,10 @@ def test_provider_with_nonexistent_path(tmp_path: Path): # Metrics should still work result = ( CheckHub() - .with_metrics([GitTrackedFileCountMetric]) + .with_metrics([GitTrackedFileCountMetric()]) .with_providers([[GitProvider(repo_path=nonexistent)]]) .measure() ) - assert len(result.metrics) == 1 - assert result.metrics[0].value == 0 + assert len(result.measurements) == 1 + assert result.measurements[0].value == 0 diff --git a/plugins/checkup-python/src/checkup_python/metrics/version.py b/plugins/checkup-python/src/checkup_python/metrics/version.py index 1173466..f475c8a 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version.py @@ -5,7 +5,6 @@ from checkup.metric import Measurement, Metric from checkup.types import Context -from checkup_python.metrics.utils import parse_semantic_version class PythonVersionMetric(Metric): @@ -82,33 +81,3 @@ def _get_runtime_version(self) -> str: """ return f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - - def __lt__(self, other) -> bool: - if not isinstance(other, PythonVersionMetric): - return NotImplemented - return parse_semantic_version(self.value) < parse_semantic_version(other.value) - - def __le__(self, other) -> bool: - if not isinstance(other, PythonVersionMetric): - return NotImplemented - return parse_semantic_version(self.value) <= parse_semantic_version(other.value) - - def __gt__(self, other) -> bool: - if not isinstance(other, PythonVersionMetric): - return NotImplemented - return parse_semantic_version(self.value) > parse_semantic_version(other.value) - - def __ge__(self, other) -> bool: - if not isinstance(other, PythonVersionMetric): - return NotImplemented - return parse_semantic_version(self.value) >= parse_semantic_version(other.value) - - def __eq__(self, other) -> bool: - if not isinstance(other, PythonVersionMetric): - return NotImplemented - return parse_semantic_version(self.value) == parse_semantic_version(other.value) - - def __ne__(self, other) -> bool: - if not isinstance(other, PythonVersionMetric): - return NotImplemented - return parse_semantic_version(self.value) != parse_semantic_version(other.value) diff --git a/plugins/checkup-python/tests/test_python_hub.py b/plugins/checkup-python/tests/test_python_hub.py index 52251cf..18a1491 100644 --- a/plugins/checkup-python/tests/test_python_hub.py +++ b/plugins/checkup-python/tests/test_python_hub.py @@ -1,18 +1,14 @@ -from pathlib import Path - from checkup_python.metrics.version_check import PythonVersionCheckMetric from checkup.hub import CheckHub def test_measure() -> None: - config_path = Path(__file__).parent / "fixtures" / "test_config.yaml" - result = ( - CheckHub(config_path=config_path) - .with_metrics([PythonVersionCheckMetric]) + CheckHub() + .with_metrics([PythonVersionCheckMetric(min_version="3.8", max_version="3.13")]) .measure() ) # `PythonVersionCheckMetric` depends on `PythonVersionMetric` - assert len(result.metrics) == 2 + assert len(result.measurements) == 2 diff --git a/plugins/checkup-python/tests/test_python_metric.py b/plugins/checkup-python/tests/test_python_metric.py index 6399364..23ed93c 100644 --- a/plugins/checkup-python/tests/test_python_metric.py +++ b/plugins/checkup-python/tests/test_python_metric.py @@ -3,19 +3,6 @@ def test_python_version_metric_calculate() -> None: metric = PythonVersionMetric() - metric.calculate(context={}, metrics={}) + measurement = metric.calculate(context={}, measurements={}) - assert metric.value is not None - - -def test_python_version_metric_compare() -> None: - metric1 = PythonVersionMetric() - metric1.value = "3.8.10" - - metric2 = PythonVersionMetric() - metric2.value = "3.9" - - assert metric1 < metric2 - assert metric2 > metric1 - assert metric1 != metric2 - assert not (metric1 == metric2) + assert measurement.value is not None diff --git a/src/checkup/materializers/console.py b/src/checkup/materializers/console.py index 1ccc353..3001e58 100644 --- a/src/checkup/materializers/console.py +++ b/src/checkup/materializers/console.py @@ -3,8 +3,8 @@ from rich.console import Console from rich.table import Table -from checkup.materializers.base import Materializer, group_metrics_by_tags -from checkup.metric import Metric +from checkup.materializers.base import Materializer, group_measurements_by_tags +from checkup.metric import Measurement class ConsoleMaterializer(Materializer): @@ -16,16 +16,20 @@ class ConsoleMaterializer(Materializer): group_tag_1: str group_tag_2: str - def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> None: - """Print metrics to console as a rich table, grouped by tags.""" - filtered = self._filter_metrics(metrics, direct_metric_names) + def materialize( + self, measurements: list[Measurement], direct_metric_names: set[str] + ) -> None: + """Print measurements to console as a rich table, grouped by tags.""" + filtered = self._filter_measurements(measurements, direct_metric_names) console = Console() - groups = group_metrics_by_tags(filtered, self.group_tag_1, self.group_tag_2) + groups = group_measurements_by_tags( + filtered, self.group_tag_1, self.group_tag_2 + ) # Create a table for each group - for (tag1_value, tag2_value), group_metrics in sorted(groups.items()): + for (tag1_value, tag2_value), group_measurements in sorted(groups.items()): table = Table( title=f"{self.group_tag_1}: {tag1_value} | {self.group_tag_2}: {tag2_value}" ) @@ -36,13 +40,13 @@ def materialize(self, metrics: list[Metric], direct_metric_names: set[str]) -> N table.add_column("Unit", style="yellow") table.add_column("Diagnostics", style="red") - for metric in group_metrics: + for measurement in group_measurements: table.add_row( - metric.name, - metric.description, - str(metric.value) if metric.value is not None else "", - metric.unit, - metric.diagnostic, + measurement.metric.name, + measurement.metric.description, + str(measurement.value) if measurement.value is not None else "", + measurement.metric.unit, + measurement.diagnostic, ) console.print(table) diff --git a/tests/test_materializers.py b/tests/test_materializers.py index 03d5d7d..8a3a7b8 100644 --- a/tests/test_materializers.py +++ b/tests/test_materializers.py @@ -26,14 +26,14 @@ def test_materializer_is_abstract(): def test_console_materializer(): """Test console output materializer.""" metric = DummyMetric(expected_value=42) - metric.value = 42 + measurement = metric.measurement(value=42) # Capture stdout captured_output = StringIO() sys.stdout = captured_output materializer = ConsoleMaterializer(group_tag_1="domain", group_tag_2="project") - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) # Reset stdout sys.stdout = sys.__stdout__ @@ -46,11 +46,11 @@ def test_console_materializer(): def test_csv_materializer(tmp_path): """Test CSV file materializer.""" metric = DummyMetric(expected_value=42) - metric.value = 42 + measurement = metric.measurement(value=42) output_file = tmp_path / "metrics.csv" materializer = CSVMaterializer(output_path=output_file) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) # Read and verify CSV content content = output_file.read_text() @@ -70,14 +70,14 @@ def test_csv_materializer_multiple_metrics(tmp_path): from fixtures import OtherDummyMetric metric1 = DummyMetric(expected_value=42) - metric1.value = 42 + measurement1 = metric1.measurement(value=42) metric2 = OtherDummyMetric(expected_value=100) - metric2.value = 100 + measurement2 = metric2.measurement(value=100) output_file = tmp_path / "metrics.csv" materializer = CSVMaterializer(output_path=output_file) - materializer.materialize([metric1, metric2], {"dummy", "other_metric"}) + materializer.materialize([measurement1, measurement2], {"dummy", "other_metric"}) content = output_file.read_text() lines = content.strip().split("\n") @@ -92,10 +92,10 @@ def test_materializer_filters_indirect_by_default(): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_metric.value = 42 + direct_measurement = direct_metric.measurement(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_metric.value = 100 + indirect_measurement = indirect_metric.measurement(value=100) # Capture stdout captured_output = StringIO() @@ -103,7 +103,7 @@ def test_materializer_filters_indirect_by_default(): materializer = ConsoleMaterializer(group_tag_1="domain", group_tag_2="project") # Only "dummy" is direct, "indirect" is not - materializer.materialize([direct_metric, indirect_metric], {"dummy"}) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) sys.stdout = sys.__stdout__ @@ -117,10 +117,10 @@ def test_materializer_includes_indirect_when_configured(): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_metric.value = 42 + direct_measurement = direct_metric.measurement(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_metric.value = 100 + indirect_measurement = indirect_metric.measurement(value=100) # Capture stdout captured_output = StringIO() @@ -129,7 +129,7 @@ def test_materializer_includes_indirect_when_configured(): materializer = ConsoleMaterializer( include_indirect=True, group_tag_1="domain", group_tag_2="project" ) - materializer.materialize([direct_metric, indirect_metric], {"dummy"}) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) sys.stdout = sys.__stdout__ @@ -143,17 +143,17 @@ def test_csv_materializer_filters_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_metric.value = 42 + direct_measurement = direct_metric.measurement(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_metric.value = 100 + indirect_measurement = indirect_metric.measurement(value=100) output_file = tmp_path / "metrics.csv" # Default: filter indirect materializer = CSVMaterializer(output_path=output_file) # Only "dummy" is direct - materializer.materialize([direct_metric, indirect_metric], {"dummy"}) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) content = output_file.read_text() lines = content.strip().split("\n") @@ -168,16 +168,16 @@ def test_csv_materializer_includes_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_metric.value = 42 + direct_measurement = direct_metric.measurement(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_metric.value = 100 + indirect_measurement = indirect_metric.measurement(value=100) output_file = tmp_path / "metrics.csv" # With include_indirect=True materializer = CSVMaterializer(output_path=output_file, include_indirect=True) - materializer.materialize([direct_metric, indirect_metric], {"dummy"}) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) content = output_file.read_text() lines = content.strip().split("\n") @@ -189,24 +189,27 @@ def test_csv_materializer_includes_indirect(tmp_path): def test_html_materializer(tmp_path): """Test HTML materializer with hierarchical grouping.""" - # Create metrics with tags + # Create measurements with tags metric1 = DummyMetric(expected_value=42) - metric1.value = 42 - metric1.tags = {"domain": "Analytics", "project": "Project A"} + measurement1 = metric1.measurement( + value=42, tags={"domain": "Analytics", "project": "Project A"} + ) metric2 = DummyMetric(expected_value=100) - metric2.value = 100 - metric2.tags = {"domain": "Analytics", "project": "Project B"} + measurement2 = metric2.measurement( + value=100, tags={"domain": "Analytics", "project": "Project B"} + ) metric3 = DummyMetric(expected_value=75) - metric3.value = 75 - metric3.tags = {"domain": "Engineering", "project": "Project C"} + measurement3 = metric3.measurement( + value=75, tags={"domain": "Engineering", "project": "Project C"} + ) output_file = tmp_path / "metrics.html" materializer = HTMLMaterializer( output_path=output_file, group_tag_1="domain", group_tag_2="project" ) - materializer.materialize([metric1, metric2, metric3], {"dummy"}) + materializer.materialize([measurement1, measurement2, measurement3], {"dummy"}) # Verify file was created assert output_file.exists() @@ -250,22 +253,26 @@ def test_html_materializer(tmp_path): def test_html_materializer_with_diagnostics(tmp_path): """Test HTML materializer with diagnostic coloring.""" - # Create metrics with different diagnostics + # Create measurements with different diagnostics metric1 = DummyMetric(expected_value=42) - metric1.value = 42 - metric1.diagnostic = "✅ All good" - metric1.tags = {"domain": "Test", "project": "TestProject"} + measurement1 = metric1.measurement( + value=42, + diagnostic="✅ All good", + tags={"domain": "Test", "project": "TestProject"}, + ) metric2 = DummyMetric(expected_value=100) - metric2.value = 100 - metric2.diagnostic = "⚠ Warning: something to check" - metric2.tags = {"domain": "Test", "project": "TestProject"} + measurement2 = metric2.measurement( + value=100, + diagnostic="⚠ Warning: something to check", + tags={"domain": "Test", "project": "TestProject"}, + ) output_file = tmp_path / "metrics.html" materializer = HTMLMaterializer( output_path=output_file, group_tag_1="domain", group_tag_2="project" ) - materializer.materialize([metric1, metric2], {"dummy"}) + materializer.materialize([measurement1, measurement2], {"dummy"}) content = output_file.read_text() @@ -279,17 +286,16 @@ def test_html_materializer_with_diagnostics(tmp_path): def test_html_materializer_ungrouped_metrics(tmp_path): - """Test HTML materializer with metrics missing tags.""" - # Create metric without tags + """Test HTML materializer with measurements missing tags.""" + # Create measurement without tags metric = DummyMetric(expected_value=42) - metric.value = 42 - metric.tags = {} # No tags + measurement = metric.measurement(value=42, tags={}) output_file = tmp_path / "metrics.html" materializer = HTMLMaterializer( output_path=output_file, group_tag_1="domain", group_tag_2="project" ) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) content = output_file.read_text() @@ -303,12 +309,14 @@ def test_html_materializer_filters_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_metric.value = 42 - direct_metric.tags = {"domain": "Test", "project": "TestProject"} + direct_measurement = direct_metric.measurement( + value=42, tags={"domain": "Test", "project": "TestProject"} + ) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_metric.value = 100 - indirect_metric.tags = {"domain": "Test", "project": "TestProject"} + indirect_measurement = indirect_metric.measurement( + value=100, tags={"domain": "Test", "project": "TestProject"} + ) output_file = tmp_path / "metrics.html" @@ -316,7 +324,7 @@ def test_html_materializer_filters_indirect(tmp_path): materializer = HTMLMaterializer( output_path=output_file, group_tag_1="domain", group_tag_2="project" ) - materializer.materialize([direct_metric, indirect_metric], {"dummy"}) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) content = output_file.read_text() @@ -328,15 +336,17 @@ def test_html_materializer_filters_indirect(tmp_path): def test_html_materializer_escape_html(tmp_path): """Test that HTML special characters are escaped.""" metric = DummyMetric(expected_value=42) - metric.value = "" - metric.diagnostic = "Test & verify " - metric.tags = {"domain": "Test & Dev", "project": "Project "} + measurement = metric.measurement( + value="", + diagnostic="Test & verify ", + tags={"domain": "Test & Dev", "project": "Project "}, + ) output_file = tmp_path / "metrics.html" materializer = HTMLMaterializer( output_path=output_file, group_tag_1="domain", group_tag_2="project" ) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) content = output_file.read_text() @@ -347,84 +357,100 @@ def test_html_materializer_escape_html(tmp_path): def test_html_materializer_end_to_end(tmp_path): - """End-to-end test with multiple metrics grouped by domain and project. + """End-to-end test with multiple measurements grouped by domain and project. This test creates a realistic scenario with multiple domains and projects, generates the HTML, and opens it for visual inspection. """ from fixtures import OtherDummyMetric - # Create metrics for Analytics domain + # Create measurements for Analytics domain metric1 = DummyMetric(expected_value=42) - metric1.value = 42 - metric1.diagnostic = "✅ Good coverage" - metric1.tags = {"domain": "Analytics", "project": "Customer Insights"} + measurement1 = metric1.measurement( + value=42, + diagnostic="✅ Good coverage", + tags={"domain": "Analytics", "project": "Customer Insights"}, + ) metric2 = OtherDummyMetric(expected_value=85) - metric2.value = 85 - metric2.diagnostic = "" - metric2.tags = {"domain": "Analytics", "project": "Customer Insights"} + measurement2 = metric2.measurement( + value=85, + diagnostic="", + tags={"domain": "Analytics", "project": "Customer Insights"}, + ) metric3 = DummyMetric(expected_value=100) - metric3.value = 100 - metric3.diagnostic = "✅ Excellent" - metric3.tags = {"domain": "Analytics", "project": "Sales Dashboard"} + measurement3 = metric3.measurement( + value=100, + diagnostic="✅ Excellent", + tags={"domain": "Analytics", "project": "Sales Dashboard"}, + ) metric4 = OtherDummyMetric(expected_value=60) - metric4.value = 60 - metric4.diagnostic = "⚠ Below target" - metric4.tags = {"domain": "Analytics", "project": "Sales Dashboard"} + measurement4 = metric4.measurement( + value=60, + diagnostic="⚠ Below target", + tags={"domain": "Analytics", "project": "Sales Dashboard"}, + ) - # Create metrics for Engineering domain + # Create measurements for Engineering domain metric5 = DummyMetric(expected_value=95) - metric5.value = 95 - metric5.diagnostic = "✅ Strong test coverage" - metric5.tags = {"domain": "Engineering", "project": "Core Platform"} + measurement5 = metric5.measurement( + value=95, + diagnostic="✅ Strong test coverage", + tags={"domain": "Engineering", "project": "Core Platform"}, + ) metric6 = OtherDummyMetric(expected_value=45) - metric6.value = 45 - metric6.diagnostic = "❌ Critical - needs attention" - metric6.tags = {"domain": "Engineering", "project": "Core Platform"} + measurement6 = metric6.measurement( + value=45, + diagnostic="❌ Critical - needs attention", + tags={"domain": "Engineering", "project": "Core Platform"}, + ) metric7 = DummyMetric(expected_value=78) - metric7.value = 78 - metric7.diagnostic = "" - metric7.tags = {"domain": "Engineering", "project": "Mobile App"} + measurement7 = metric7.measurement( + value=78, diagnostic="", tags={"domain": "Engineering", "project": "Mobile App"} + ) - # Create metrics for Data Science domain + # Create measurements for Data Science domain metric8 = DummyMetric(expected_value=92) - metric8.value = 92 - metric8.diagnostic = "✅ Model accuracy within range" - metric8.tags = {"domain": "Data Science", "project": "ML Pipeline"} + measurement8 = metric8.measurement( + value=92, + diagnostic="✅ Model accuracy within range", + tags={"domain": "Data Science", "project": "ML Pipeline"}, + ) metric9 = OtherDummyMetric(expected_value=88) - metric9.value = 88 - metric9.diagnostic = "" - metric9.tags = {"domain": "Data Science", "project": "ML Pipeline"} + measurement9 = metric9.measurement( + value=88, + diagnostic="", + tags={"domain": "Data Science", "project": "ML Pipeline"}, + ) metric10 = DummyMetric(expected_value=55) - metric10.value = 55 - metric10.diagnostic = "⚠ Training data quality concerns" - metric10.tags = {"domain": "Data Science", "project": "Recommendation Engine"} + measurement10 = metric10.measurement( + value=55, + diagnostic="⚠ Training data quality concerns", + tags={"domain": "Data Science", "project": "Recommendation Engine"}, + ) - # Create some ungrouped metrics + # Create some ungrouped measurements metric11 = DummyMetric(expected_value=70) - metric11.value = 70 - metric11.diagnostic = "" - metric11.tags = {} # No tags - - all_metrics = [ - metric1, - metric2, - metric3, - metric4, - metric5, - metric6, - metric7, - metric8, - metric9, - metric10, - metric11, + measurement11 = metric11.measurement(value=70, diagnostic="", tags={}) + + all_measurements = [ + measurement1, + measurement2, + measurement3, + measurement4, + measurement5, + measurement6, + measurement7, + measurement8, + measurement9, + measurement10, + measurement11, ] direct_names = {"dummy", "other_metric"} @@ -436,7 +462,7 @@ def test_html_materializer_end_to_end(tmp_path): group_tag_2="project", include_indirect=False, ) - materializer.materialize(all_metrics, direct_names) + materializer.materialize(all_measurements, direct_names) # Verify file was created assert output_file.exists() @@ -472,16 +498,15 @@ def test_html_materializer_end_to_end(tmp_path): def test_sqlalchemy_materializer(tmp_path): - """Test SQLAlchemy materializer writes metrics to database.""" + """Test SQLAlchemy materializer writes measurements to database.""" metric = DummyMetric(expected_value=42) - metric.value = 42 - metric.tags = {"domain": "Analytics"} + measurement = metric.measurement(value=42, tags={"domain": "Analytics"}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", ) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) # Verify data was written engine = create_engine(f"sqlite:///{db_path}") @@ -498,20 +523,20 @@ def test_sqlalchemy_materializer(tmp_path): def test_sqlalchemy_materializer_multiple_metrics(tmp_path): - """Test SQLAlchemy materializer with multiple metrics.""" + """Test SQLAlchemy materializer with multiple measurements.""" from fixtures import OtherDummyMetric metric1 = DummyMetric(expected_value=42) - metric1.value = 42 + measurement1 = metric1.measurement(value=42) metric2 = OtherDummyMetric(expected_value=100) - metric2.value = 100 + measurement2 = metric2.measurement(value=100) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", ) - materializer.materialize([metric1, metric2], {"dummy", "other_metric"}) + materializer.materialize([measurement1, measurement2], {"dummy", "other_metric"}) engine = create_engine(f"sqlite:///{db_path}") with engine.connect() as conn: @@ -525,15 +550,15 @@ def test_sqlalchemy_materializer_multiple_metrics(tmp_path): def test_sqlalchemy_materializer_appends_rows(tmp_path): """Test that successive materializations append rows.""" metric = DummyMetric(expected_value=42) - metric.value = 42 + measurement = metric.measurement(value=42) db_path = tmp_path / "metrics.db" url = f"sqlite:///{db_path}" materializer = SQLAlchemyMaterializer(connection_url=url) # Materialize twice - materializer.materialize([metric], {"dummy"}) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) engine = create_engine(url) with engine.connect() as conn: @@ -545,14 +570,14 @@ def test_sqlalchemy_materializer_appends_rows(tmp_path): def test_sqlalchemy_materializer_custom_table_name(tmp_path): """Test SQLAlchemy materializer with custom table name.""" metric = DummyMetric(expected_value=42) - metric.value = 42 + measurement = metric.measurement(value=42) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", table_name="checkup_results", ) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) engine = create_engine(f"sqlite:///{db_path}") with engine.connect() as conn: @@ -563,20 +588,20 @@ def test_sqlalchemy_materializer_custom_table_name(tmp_path): def test_sqlalchemy_materializer_filters_indirect(tmp_path): - """Test SQLAlchemy materializer filtering of indirect metrics.""" + """Test SQLAlchemy materializer filtering of indirect measurements.""" from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_metric.value = 42 + direct_measurement = direct_metric.measurement(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_metric.value = 100 + indirect_measurement = indirect_metric.measurement(value=100) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", ) - materializer.materialize([direct_metric, indirect_metric], {"dummy"}) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) engine = create_engine(f"sqlite:///{db_path}") with engine.connect() as conn: @@ -587,21 +612,21 @@ def test_sqlalchemy_materializer_filters_indirect(tmp_path): def test_sqlalchemy_materializer_includes_indirect(tmp_path): - """Test SQLAlchemy materializer including indirect metrics.""" + """Test SQLAlchemy materializer including indirect measurements.""" from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_metric.value = 42 + direct_measurement = direct_metric.measurement(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_metric.value = 100 + indirect_measurement = indirect_metric.measurement(value=100) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", include_indirect=True, ) - materializer.materialize([direct_metric, indirect_metric], {"dummy"}) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) engine = create_engine(f"sqlite:///{db_path}") with engine.connect() as conn: @@ -615,14 +640,13 @@ def test_sqlalchemy_materializer_includes_indirect(tmp_path): def test_sqlalchemy_materializer_none_value(tmp_path): """Test SQLAlchemy materializer handles None values.""" metric = DummyMetric(expected_value=42) - # value is None (not calculated) - metric.tags = {} + measurement = metric.measurement(value=None, tags={}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", ) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) engine = create_engine(f"sqlite:///{db_path}") with engine.connect() as conn: @@ -678,16 +702,17 @@ def test_sqlalchemy_materializer_expand_tags(tmp_path): from fixtures import IndirectDummyMetric, OtherDummyMetric metric1 = DummyMetric(expected_value=42) - metric1.value = 42 - metric1.tags = {"domain": "Analytics", "env": "prod"} + measurement1 = metric1.measurement( + value=42, tags={"domain": "Analytics", "env": "prod"} + ) metric2 = OtherDummyMetric(expected_value=100) - metric2.value = 100 - metric2.tags = {"domain": "Engineering", "team": "platform"} + measurement2 = metric2.measurement( + value=100, tags={"domain": "Engineering", "team": "platform"} + ) metric3 = IndirectDummyMetric(expected_value=50) - metric3.value = 50 - metric3.tags = None # No tags + measurement3 = metric3.measurement(value=50, tags={}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -695,7 +720,9 @@ def test_sqlalchemy_materializer_expand_tags(tmp_path): expand_tags=True, include_indirect=True, ) - materializer.materialize([metric1, metric2, metric3], {"dummy", "other_metric"}) + materializer.materialize( + [measurement1, measurement2, measurement3], {"dummy", "other_metric"} + ) engine = create_engine(f"sqlite:///{db_path}") with engine.connect() as conn: @@ -721,7 +748,7 @@ def test_sqlalchemy_materializer_expand_tags(tmp_path): assert rows[0][0] == "dummy" assert rows[0][1] == "Analytics" assert rows[0][2] == "prod" - assert rows[0][3] is None # team not in metric1 + assert rows[0][3] is None # team not in measurement1 # indirect metric (no tags) assert rows[1][0] == "indirect" @@ -732,22 +759,21 @@ def test_sqlalchemy_materializer_expand_tags(tmp_path): # other_metric assert rows[2][0] == "other_metric" assert rows[2][1] == "Engineering" - assert rows[2][2] is None # env not in metric2 + assert rows[2][2] is None # env not in measurement2 assert rows[2][3] == "platform" def test_sqlalchemy_materializer_expand_tags_no_tags(tmp_path): - """Test expand_tags handles metrics with no tags.""" + """Test expand_tags handles measurements with no tags.""" metric = DummyMetric(expected_value=42) - metric.value = 42 - metric.tags = None + measurement = metric.measurement(value=42, tags={}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", expand_tags=True, ) - materializer.materialize([metric], {"dummy"}) + materializer.materialize([measurement], {"dummy"}) engine = create_engine(f"sqlite:///{db_path}") with engine.connect() as conn: @@ -766,19 +792,19 @@ def test_sqlalchemy_materializer_expand_tags_no_tags(tmp_path): def test_sqlalchemy_materializer_batch_size(tmp_path): """Test SQLAlchemy materializer respects batch_size for large inserts.""" - # Create more metrics than the batch size - metrics = [] + # Create more measurements than the batch size + measurements = [] for i in range(25): metric = DummyMetric(expected_value=i) - metric.value = i - metrics.append(metric) + measurement = metric.measurement(value=i) + measurements.append(measurement) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( connection_url=f"sqlite:///{db_path}", batch_size=10, # Small batch size to test batching ) - materializer.materialize(metrics, {"dummy"}) + materializer.materialize(measurements, {"dummy"}) # Verify all rows were inserted engine = create_engine(f"sqlite:///{db_path}") From b06a6dd5251db231edd6ccb6752d7f46ed592266 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 09:03:19 +0200 Subject: [PATCH 09/21] PR coment and rebase on metric/measurement split --- src/checkup/materializers/console.py | 136 ++++++++++++++++++++------- tests/test_materializers.py | 55 ++++++++++- 2 files changed, 155 insertions(+), 36 deletions(-) diff --git a/src/checkup/materializers/console.py b/src/checkup/materializers/console.py index 5167d05..def8572 100644 --- a/src/checkup/materializers/console.py +++ b/src/checkup/materializers/console.py @@ -1,54 +1,126 @@ """Console materializer for terminal output.""" +from pydantic import field_validator from rich.console import Console from rich.table import Table -from checkup.materializers.base import Materializer, group_measurements_by_tags +from checkup.materializers.base import Materializer from checkup.metric import Measurement class ConsoleMaterializer(Materializer): - """Output metrics to console. + """ + Output measurements to console. + + Outputs a rich table with measurement details. + Optionally groups measurements by tag values (max 2 levels). - Outputs a rich table with metric details. - Optionally groups metrics by tag values. + Args: + group_tags: List of tag names to group by (max 2). If empty, no grouping. + include_indirect: If True, include indirect measurements. """ - group_tag_1: str | None = None - group_tag_2: str | None = None + group_tags: list[str] = [] + + @field_validator("group_tags") + @classmethod + def validate_group_tags(cls, v: list[str]) -> list[str]: + if len(v) > 2: + raise ValueError(f"Maximum 2 group tags supported, got {len(v)}: {v}") + return v def materialize( self, measurements: list[Measurement], direct_metric_names: set[str] ) -> None: - """Print measurements to console as a rich table, grouped by tags.""" - filtered = self._filter_measurements(measurements, direct_metric_names) + """ + Print measurements to console as a rich table, optionally grouped by tags. + """ + filtered = self._filter_measurements(measurements, direct_metric_names) console = Console() - groups = group_measurements_by_tags( - filtered, self.group_tag_1, self.group_tag_2 - ) + if not self.group_tags: + self._print_table(console, filtered, title=None) + elif len(self.group_tags) == 1: + groups = self._group_by_single_tag(filtered, self.group_tags[0]) + for tag_value, group_measurements in sorted(groups.items()): + title = f"{self.group_tags[0]}: {tag_value}" + self._print_table(console, group_measurements, title=title) + console.print() + else: + groups = self._group_by_two_tags( + filtered, self.group_tags[0], self.group_tags[1] + ) + for (tag1_value, tag2_value), group_measurements in sorted(groups.items()): + title = f"{self.group_tags[0]}: {tag1_value} | {self.group_tags[1]}: {tag2_value}" + self._print_table(console, group_measurements, title=title) + console.print() + + def _print_table( + self, + console: Console, + measurements: list[Measurement], + title: str | None, + ) -> None: + """ + Print a single table of measurements. + """ - # Create a table for each group - for (tag1_value, tag2_value), group_measurements in sorted(groups.items()): - table = Table( - title=f"{self.group_tag_1}: {tag1_value} | {self.group_tag_2}: {tag2_value}" + table = Table(title=title) + + table.add_column("Name", style="cyan", no_wrap=True) + table.add_column("Description", style="dim") + table.add_column("Value", justify="right", style="green") + table.add_column("Unit", style="yellow") + table.add_column("Diagnostics", style="red") + + for measurement in measurements: + table.add_row( + measurement.metric.name, + measurement.metric.description, + str(measurement.value) if measurement.value is not None else "", + measurement.metric.unit, + measurement.diagnostic, ) - table.add_column("Name", style="cyan", no_wrap=True) - table.add_column("Description", style="dim") - table.add_column("Value", justify="right", style="green") - table.add_column("Unit", style="yellow") - table.add_column("Diagnostics", style="red") - - for measurement in group_measurements: - table.add_row( - measurement.metric.name, - measurement.metric.description, - str(measurement.value) if measurement.value is not None else "", - measurement.metric.unit, - measurement.diagnostic, - ) - - console.print(table) - console.print() # Add spacing between tables + console.print(table) + + def _group_by_single_tag( + self, + measurements: list[Measurement], + tag: str, + default: str = "Unknown", + ) -> dict[str, list[Measurement]]: + """ + Group measurements by a single tag value. + """ + + groups: dict[str, list[Measurement]] = {} + for measurement in measurements: + key = measurement.tags.get(tag, default) + if key not in groups: + groups[key] = [] + groups[key].append(measurement) + return groups + + def _group_by_two_tags( + self, + measurements: list[Measurement], + tag1: str, + tag2: str, + default: str = "Unknown", + ) -> dict[tuple[str, str], list[Measurement]]: + """ + Group measurements by two tag values. + """ + + groups: dict[tuple[str, str], list[Measurement]] = {} + for measurement in measurements: + key = ( + measurement.tags.get(tag1, default), + measurement.tags.get(tag2, default), + ) + if key not in groups: + groups[key] = [] + groups[key].append(measurement) + return groups diff --git a/tests/test_materializers.py b/tests/test_materializers.py index 8a3a7b8..241364d 100644 --- a/tests/test_materializers.py +++ b/tests/test_materializers.py @@ -24,7 +24,7 @@ def test_materializer_is_abstract(): def test_console_materializer(): - """Test console output materializer.""" + """Test console output materializer with two-level grouping.""" metric = DummyMetric(expected_value=42) measurement = metric.measurement(value=42) @@ -32,7 +32,7 @@ def test_console_materializer(): captured_output = StringIO() sys.stdout = captured_output - materializer = ConsoleMaterializer(group_tag_1="domain", group_tag_2="project") + materializer = ConsoleMaterializer(group_tags=["domain", "project"]) materializer.materialize([measurement], {"dummy"}) # Reset stdout @@ -43,6 +43,53 @@ def test_console_materializer(): assert "42" in output +def test_console_materializer_no_grouping(): + """Test console materializer without grouping.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measurement(value=42) + + captured_output = StringIO() + sys.stdout = captured_output + + materializer = ConsoleMaterializer() # No group_tags + materializer.materialize([measurement], {"dummy"}) + + sys.stdout = sys.__stdout__ + + output = captured_output.getvalue() + assert "dummy" in output + assert "42" in output + + +def test_console_materializer_single_grouping(): + """Test console materializer with single-level grouping.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measurement(value=42, tags={"domain": "Analytics"}) + + captured_output = StringIO() + sys.stdout = captured_output + + materializer = ConsoleMaterializer(group_tags=["domain"]) + materializer.materialize([measurement], {"dummy"}) + + sys.stdout = sys.__stdout__ + + output = captured_output.getvalue() + assert "dummy" in output + assert "42" in output + assert "domain: Analytics" in output + + +def test_console_materializer_too_many_group_tags(): + """Test that more than 2 group tags raises an error.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError) as exc_info: + ConsoleMaterializer(group_tags=["a", "b", "c"]) + + assert "Maximum 2 group tags supported" in str(exc_info.value) + + def test_csv_materializer(tmp_path): """Test CSV file materializer.""" metric = DummyMetric(expected_value=42) @@ -101,7 +148,7 @@ def test_materializer_filters_indirect_by_default(): captured_output = StringIO() sys.stdout = captured_output - materializer = ConsoleMaterializer(group_tag_1="domain", group_tag_2="project") + materializer = ConsoleMaterializer(group_tags=["domain", "project"]) # Only "dummy" is direct, "indirect" is not materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) @@ -127,7 +174,7 @@ def test_materializer_includes_indirect_when_configured(): sys.stdout = captured_output materializer = ConsoleMaterializer( - include_indirect=True, group_tag_1="domain", group_tag_2="project" + include_indirect=True, group_tags=["domain", "project"] ) materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) From f495bc3afc12696705b4edb810075ff2e1d0cf3e Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 09:14:07 +0200 Subject: [PATCH 10/21] update grouping --- src/checkup/materializers/console.py | 62 ++++++---------------------- tests/test_materializers.py | 25 ++++++++--- 2 files changed, 32 insertions(+), 55 deletions(-) diff --git a/src/checkup/materializers/console.py b/src/checkup/materializers/console.py index def8572..1951a5b 100644 --- a/src/checkup/materializers/console.py +++ b/src/checkup/materializers/console.py @@ -1,6 +1,5 @@ """Console materializer for terminal output.""" -from pydantic import field_validator from rich.console import Console from rich.table import Table @@ -13,22 +12,15 @@ class ConsoleMaterializer(Materializer): Output measurements to console. Outputs a rich table with measurement details. - Optionally groups measurements by tag values (max 2 levels). + Optionally groups measurements by tag values. Args: - group_tags: List of tag names to group by (max 2). If empty, no grouping. + group_tags: List of tag names to group by. If empty, no grouping. include_indirect: If True, include indirect measurements. """ group_tags: list[str] = [] - @field_validator("group_tags") - @classmethod - def validate_group_tags(cls, v: list[str]) -> list[str]: - if len(v) > 2: - raise ValueError(f"Maximum 2 group tags supported, got {len(v)}: {v}") - return v - def materialize( self, measurements: list[Measurement], direct_metric_names: set[str] ) -> None: @@ -41,18 +33,13 @@ def materialize( if not self.group_tags: self._print_table(console, filtered, title=None) - elif len(self.group_tags) == 1: - groups = self._group_by_single_tag(filtered, self.group_tags[0]) - for tag_value, group_measurements in sorted(groups.items()): - title = f"{self.group_tags[0]}: {tag_value}" - self._print_table(console, group_measurements, title=title) - console.print() else: - groups = self._group_by_two_tags( - filtered, self.group_tags[0], self.group_tags[1] - ) - for (tag1_value, tag2_value), group_measurements in sorted(groups.items()): - title = f"{self.group_tags[0]}: {tag1_value} | {self.group_tags[1]}: {tag2_value}" + groups = self._group_by_tags(filtered) + for tag_values, group_measurements in sorted(groups.items()): + title = " | ".join( + f"{tag}: {value}" + for tag, value in zip(self.group_tags, tag_values, strict=True) + ) self._print_table(console, group_measurements, title=title) console.print() @@ -85,41 +72,18 @@ def _print_table( console.print(table) - def _group_by_single_tag( + def _group_by_tags( self, measurements: list[Measurement], - tag: str, default: str = "Unknown", - ) -> dict[str, list[Measurement]]: + ) -> dict[tuple[str, ...], list[Measurement]]: """ - Group measurements by a single tag value. + Group measurements by tag values. """ - groups: dict[str, list[Measurement]] = {} + groups: dict[tuple[str, ...], list[Measurement]] = {} for measurement in measurements: - key = measurement.tags.get(tag, default) - if key not in groups: - groups[key] = [] - groups[key].append(measurement) - return groups - - def _group_by_two_tags( - self, - measurements: list[Measurement], - tag1: str, - tag2: str, - default: str = "Unknown", - ) -> dict[tuple[str, str], list[Measurement]]: - """ - Group measurements by two tag values. - """ - - groups: dict[tuple[str, str], list[Measurement]] = {} - for measurement in measurements: - key = ( - measurement.tags.get(tag1, default), - measurement.tags.get(tag2, default), - ) + key = tuple(measurement.tags.get(tag, default) for tag in self.group_tags) if key not in groups: groups[key] = [] groups[key].append(measurement) diff --git a/tests/test_materializers.py b/tests/test_materializers.py index 241364d..4d80562 100644 --- a/tests/test_materializers.py +++ b/tests/test_materializers.py @@ -80,14 +80,27 @@ def test_console_materializer_single_grouping(): assert "domain: Analytics" in output -def test_console_materializer_too_many_group_tags(): - """Test that more than 2 group tags raises an error.""" - from pydantic import ValidationError +def test_console_materializer_three_level_grouping(): + """Test console materializer with three-level grouping.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measurement( + value=42, tags={"domain": "Analytics", "project": "Core", "env": "prod"} + ) - with pytest.raises(ValidationError) as exc_info: - ConsoleMaterializer(group_tags=["a", "b", "c"]) + captured_output = StringIO() + sys.stdout = captured_output + + materializer = ConsoleMaterializer(group_tags=["domain", "project", "env"]) + materializer.materialize([measurement], {"dummy"}) - assert "Maximum 2 group tags supported" in str(exc_info.value) + sys.stdout = sys.__stdout__ + + output = captured_output.getvalue() + assert "dummy" in output + assert "42" in output + assert "domain: Analytics" in output + assert "project: Core" in output + assert "env: prod" in output def test_csv_materializer(tmp_path): From 8256abd5088742ec0daa71843af67b482d06d15e Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 09:59:11 +0200 Subject: [PATCH 11/21] update --- checkup.schema.json | 208 +++++++++++++++++----------- pyproject.toml | 7 + src/checkup/__init__.py | 8 +- src/checkup/cli/__init__.py | 8 -- src/checkup/cli/executor.py | 23 ++- src/checkup/configuration/schema.py | 137 +++++++----------- tests/test_cli_configuration.py | 36 ----- uv.lock | 35 +++++ 8 files changed, 236 insertions(+), 226 deletions(-) diff --git a/checkup.schema.json b/checkup.schema.json index 41d1c87..f418c89 100644 --- a/checkup.schema.json +++ b/checkup.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://checkup.dev/schemas/checkup.yaml.json", "title": "Checkup Configuration", - "description": "Configuration file for checkup data product health metrics", + "description": "Configuration file for checkup", "type": "object", "properties": { "tags": { @@ -530,101 +530,147 @@ } }, "materializer": { - "type": "object", - "description": "Output materializer configuration", - "properties": { - "type": { - "type": "string", - "enum": [ - "console", - "csv", - "html", - "sqlalchemy" - ] - }, - "include_indirect": { - "default": false, - "title": "Include Indirect", - "type": "boolean" - }, - "group_tag_1": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "console" + }, + "include_indirect": { + "default": false, + "title": "Include Indirect", + "type": "boolean" + }, + "group_tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Group Tags", + "type": "array" } + }, + "required": [ + "type" ], - "default": null, - "title": "Group Tag 1" + "additionalProperties": false }, - "group_tag_2": { - "anyOf": [ - { + { + "type": "object", + "properties": { + "type": { + "const": "csv" + }, + "include_indirect": { + "default": false, + "title": "Include Indirect", + "type": "boolean" + }, + "output_path": { + "format": "path", + "title": "Output Path", "type": "string" - }, - { - "type": "null" } + }, + "required": [ + "type" ], - "default": null, - "title": "Group Tag 2" - }, - "output_path": { - "format": "path", - "title": "Output Path", - "type": "string" - }, - "connection_url": { - "format": "password", - "title": "Connection Url", - "type": "string", - "writeOnly": true - }, - "table_name": { - "default": "metrics", - "title": "Table Name", - "type": "string" + "additionalProperties": false }, - "table_schema": { - "anyOf": [ - { + { + "type": "object", + "properties": { + "type": { + "const": "html" + }, + "include_indirect": { + "default": false, + "title": "Include Indirect", + "type": "boolean" + }, + "output_path": { + "format": "path", + "title": "Output Path", "type": "string" }, - { - "type": "null" + "group_tag_1": { + "title": "Group Tag 1", + "type": "string" + }, + "group_tag_2": { + "title": "Group Tag 2", + "type": "string" } + }, + "required": [ + "type" ], - "default": null, - "title": "Table Schema" + "additionalProperties": false }, - "connect_args": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" + { + "type": "object", + "properties": { + "type": { + "const": "sqlalchemy" + }, + "include_indirect": { + "default": false, + "title": "Include Indirect", + "type": "boolean" + }, + "connection_url": { + "format": "password", + "title": "Connection Url", + "type": "string", + "writeOnly": true + }, + "table_name": { + "default": "metrics", + "title": "Table Name", + "type": "string" + }, + "table_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Table Schema" + }, + "connect_args": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Connect Args" + }, + "expand_tags": { + "default": false, + "title": "Expand Tags", + "type": "boolean" + }, + "batch_size": { + "default": 1000, + "title": "Batch Size", + "type": "integer" } + }, + "required": [ + "type" ], - "default": null, - "title": "Connect Args" - }, - "expand_tags": { - "default": false, - "title": "Expand Tags", - "type": "boolean" - }, - "batch_size": { - "default": 1000, - "title": "Batch Size", - "type": "integer" + "additionalProperties": false } - }, - "required": [ - "type" ] } }, diff --git a/pyproject.toml b/pyproject.toml index 79b8ad0..6eb06fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,11 +16,18 @@ dependencies = [ "pyyaml>=6.0", "rich>=13.0", "sqlalchemy>=2.0", + "typer>=0.24", ] [project.scripts] checkup = "checkup:main" +[project.entry-points."checkup.materializers"] +console = "checkup.materializers:ConsoleMaterializer" +csv = "checkup.materializers:CSVMaterializer" +html = "checkup.materializers:HTMLMaterializer" +sqlalchemy = "checkup.materializers:SQLAlchemyMaterializer" + [build-system] requires = ["uv_build>=0.8.13,<0.9.0"] build-backend = "uv_build" diff --git a/src/checkup/__init__.py b/src/checkup/__init__.py index 842a796..0ca9cf5 100644 --- a/src/checkup/__init__.py +++ b/src/checkup/__init__.py @@ -1,4 +1,4 @@ -"""Checkup - Extensible metrics calculation framework.""" +"""Checkup - Computational governance framework for measuring data product health.""" from checkup.errors import ( DuplicateMetricNameError, @@ -49,6 +49,6 @@ def main() -> None: - """CLI entry point.""" - print("Checkup metrics framework") - print("Import CheckHub to get started") + from checkup.cli import app + + app() diff --git a/src/checkup/cli/__init__.py b/src/checkup/cli/__init__.py index 6b3be8c..15ceaa4 100644 --- a/src/checkup/cli/__init__.py +++ b/src/checkup/cli/__init__.py @@ -17,11 +17,3 @@ app.command()(init) app.command()(config) app.command()(schema) - - -def main() -> None: - """ - CLI entry point. - """ - - app() diff --git a/src/checkup/cli/executor.py b/src/checkup/cli/executor.py index 6dcea94..89424d2 100644 --- a/src/checkup/cli/executor.py +++ b/src/checkup/cli/executor.py @@ -89,15 +89,12 @@ def _resolve_providers( def _resolve_metrics( config: CheckupConfig, registry: "PluginRegistry", -) -> list[type["Metric"]]: +) -> list["Metric"]: """ - Resolve metric configs to metric classes. - - If a metric has config, a subclass is created with those defaults. + Resolve metric configs to metric instances. """ - from checkup.metric import create_configured_metric - metrics: list[type[Metric]] = [] + metrics: list[Metric] = [] for metric_config in config.metrics: metric_cls = registry.get_metric(metric_config.name) @@ -105,10 +102,13 @@ def _resolve_metrics( console.print(f"[yellow]Unknown metric: {metric_config.name}[/yellow]") continue - if metric_config.config: - metric_cls = create_configured_metric(metric_cls, metric_config.config) - - metrics.append(metric_cls) + try: + metric = metric_cls(**metric_config.config) + metrics.append(metric) + except Exception as e: + console.print( + f"[red]Failed to instantiate metric {metric_config.name}: {e}[/red]" + ) return metrics @@ -122,7 +122,6 @@ def _resolve_materializer( Resolve materializer config to materializer instance. """ - # Determine type and config if override: mat_type = override mat_config = {} @@ -133,11 +132,9 @@ def _resolve_materializer( mat_type = "console" mat_config = {} - # Get materializer class materializer_cls = registry.get_materializer(mat_type) if materializer_cls is None: - # Fallback to built-in console console.print( f"[yellow]Unknown materializer: {mat_type}, using console[/yellow]" ) diff --git a/src/checkup/configuration/schema.py b/src/checkup/configuration/schema.py index 062e3ed..990b9d1 100644 --- a/src/checkup/configuration/schema.py +++ b/src/checkup/configuration/schema.py @@ -113,94 +113,66 @@ def _get_provider_schema(cls: type) -> dict | None: return schema -def generate_schema() -> dict: +def _build_oneof_schema( + names: list[str], + schemas: dict[str, dict], + key_field: str = "name", +) -> dict[str, Any]: """ - Generate JSON schema for checkup.yaml configuration. - - Dynamically includes available providers, metrics, and materializers - using Pydantic's schema generation. + Build a oneOf schema for a list of named items. """ - registry = get_registry() - - # Collect provider info - provider_names = sorted(registry.providers.keys()) - provider_schemas = {} - for name, cls in registry.providers.items(): - schema = _get_provider_schema(cls) - if schema: - provider_schemas[name] = schema - - # Collect metric info - metric_names = sorted(registry.metrics.keys()) - metric_schemas = {} - for name, cls in registry.metrics.items(): - schema = _get_pydantic_schema(cls) - if schema: - metric_schemas[name] = schema - - # Collect materializer info - materializer_names = sorted(registry.materializers.keys()) - materializer_schemas = {} - for name, cls in registry.materializers.items(): - schema = _get_pydantic_schema(cls) - if schema: - materializer_schemas[name] = schema - - # Build provider item schema with oneOf for each provider - provider_variants = [] - for name in provider_names: + variants = [] + for name in names: variant: dict[str, Any] = { "type": "object", - "properties": { - "name": {"const": name}, - }, - "required": ["name"], + "properties": {key_field: {"const": name}}, + "required": [key_field], "additionalProperties": False, } - if name in provider_schemas and "properties" in provider_schemas[name]: - variant["properties"].update(provider_schemas[name]["properties"]) - provider_variants.append(variant) + if name in schemas and "properties" in schemas[name]: + variant["properties"].update(schemas[name]["properties"]) + variants.append(variant) - provider_item_schema: dict[str, Any] = ( - {"oneOf": provider_variants} if provider_variants else {"type": "object"} - ) + return {"oneOf": variants} if variants else {"type": "object"} - # Build metric item schema with oneOf for each metric - metric_variants = [] - for name in metric_names: - variant: dict[str, Any] = { - "type": "object", - "properties": { - "name": {"const": name}, - }, - "required": ["name"], - "additionalProperties": False, - } - if name in metric_schemas and "properties" in metric_schemas[name]: - variant["properties"].update(metric_schemas[name]["properties"]) - metric_variants.append(variant) - metric_item_schema: dict[str, Any] = ( - {"oneOf": metric_variants} if metric_variants else {"type": "object"} - ) +def _collect_schemas( + items: dict[str, type], + schema_fn: callable, +) -> tuple[list[str], dict[str, dict]]: + """ + Collect schemas for a dict of named classes. + """ - # Build materializer schema - materializer_props: dict[str, Any] = { - "type": { - "type": "string", - "enum": materializer_names, - } - if materializer_names - else {"type": "string"} - } + names = sorted(items.keys()) + schemas = {} + for name, cls in items.items(): + schema = schema_fn(cls) + if schema: + schemas[name] = schema + return names, schemas - # Add properties from all materializers - for schema in materializer_schemas.values(): - if "properties" in schema: - for prop_name, prop_schema in schema["properties"].items(): - if prop_name not in materializer_props: - materializer_props[prop_name] = prop_schema + +def generate_schema() -> dict: + """ + Generate JSON schema for checkup.yaml configuration. + + Dynamically includes available providers, metrics, and materializers + using Pydantic's schema generation. + """ + + registry = get_registry() + + provider_names, provider_schemas = _collect_schemas( + registry.providers, _get_provider_schema + ) + metric_names, metric_schemas = _collect_schemas( + registry.metrics, _get_pydantic_schema + ) + materializer_names, materializer_schemas = _collect_schemas( + registry.materializers, _get_pydantic_schema + ) return { "$schema": SCHEMA_VERSION, @@ -217,19 +189,16 @@ def generate_schema() -> dict: "providers": { "type": "array", "description": "Data providers for context enrichment", - "items": provider_item_schema, + "items": _build_oneof_schema(provider_names, provider_schemas), }, "metrics": { "type": "array", "description": "Metrics to calculate", - "items": metric_item_schema, - }, - "materializer": { - "type": "object", - "description": "Output materializer configuration", - "properties": materializer_props, - "required": ["type"], + "items": _build_oneof_schema(metric_names, metric_schemas), }, + "materializer": _build_oneof_schema( + materializer_names, materializer_schemas, key_field="type" + ), }, "additionalProperties": False, } diff --git a/tests/test_cli_configuration.py b/tests/test_cli_configuration.py index 8a7a4c6..7e87e87 100644 --- a/tests/test_cli_configuration.py +++ b/tests/test_cli_configuration.py @@ -2,8 +2,6 @@ Tests for CLI configuration loading and parsing. """ -from typing import ClassVar - import yaml from checkup.configuration.env import ( @@ -18,8 +16,6 @@ parse_providers, ) from checkup.configuration.models import CheckupConfig -from checkup.metric import Metric -from checkup.types import Context class TestParseProviders: @@ -254,35 +250,3 @@ def test_hierarchical_loading_merges_parent_and_child(self, tmp_path): result = load_config(start_dir=child_dir) assert result.tags == {"team": "platform", "product": "my-product"} - - -class ConfigurableMetric(Metric): - """Test metric with required config fields.""" - - name: ClassVar[str] = "configurable" - description: ClassVar[str] = "Needs config" - unit: ClassVar[str] = "count" - - multiplier: int - offset: int = 0 - - def calculate(self, _context: Context, _metrics: dict) -> None: - self.value = 10 * self.multiplier + self.offset - - -class TestHubMetricConfigs: - def test_configured_subclass_passed_to_hub(self): - from checkup.hub import CheckHub - from checkup.metric import create_configured_metric - - # Create a configured subclass (this is how CLI executor does it) - Configured = create_configured_metric( - ConfigurableMetric, - {"multiplier": 5, "offset": 3}, - ) - - hub = CheckHub().with_metrics([Configured]) - result = hub.measure() - - assert len(result.metrics) == 1 - assert result.metrics[0].value == 53 # 10 * 5 + 3 diff --git a/uv.lock b/uv.lock index e2bc4ae..8696309 100644 --- a/uv.lock +++ b/uv.lock @@ -33,6 +33,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/53/89b197cb472a3175d73384761a3413fd58e6b65a794c1102d148b8de87bd/agate-1.9.1-py2.py3-none-any.whl", hash = "sha256:1cf329510b3dde07c4ad1740b7587c9c679abc3dcd92bb1107eabc10c2e03c50", size = 95085, upload-time = "2023-12-21T20:05:21.954Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -150,6 +159,7 @@ dependencies = [ { name = "pyyaml" }, { name = "rich" }, { name = "sqlalchemy" }, + { name = "typer" }, ] [package.dev-dependencies] @@ -172,6 +182,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0" }, { name = "rich", specifier = ">=13.0" }, { name = "sqlalchemy", specifier = ">=2.0" }, + { name = "typer", specifier = ">=0.24" }, ] [package.metadata.requires-dev] @@ -1506,6 +1517,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -1588,6 +1608,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "typer" +version = "0.24.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/24/cb09efec5cc954f7f9b930bf8279447d24618bb6758d4f6adf2574c41780/typer-0.24.1.tar.gz", hash = "sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45", size = 118613, upload-time = "2026-02-21T16:54:40.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/91/48db081e7a63bb37284f9fbcefda7c44c277b18b0e13fbc36ea2335b71e6/typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e", size = 56085, upload-time = "2026-02-21T16:54:41.616Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 7beda35aef885b2178e677545c9c28ae6c6f364b Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 10:04:44 +0200 Subject: [PATCH 12/21] dependency --- pyproject.toml | 1 + uv.lock | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6eb06fb..6d9da26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "jinja2>=3.1.6", "pydantic>=2.11.7", "pyyaml>=6.0", + "questionary>=2.0", "rich>=13.0", "sqlalchemy>=2.0", "typer>=0.24", diff --git a/uv.lock b/uv.lock index 8696309..caad284 100644 --- a/uv.lock +++ b/uv.lock @@ -157,6 +157,7 @@ dependencies = [ { name = "jinja2" }, { name = "pydantic" }, { name = "pyyaml" }, + { name = "questionary" }, { name = "rich" }, { name = "sqlalchemy" }, { name = "typer" }, @@ -180,6 +181,7 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.6" }, { name = "pydantic", specifier = ">=2.11.7" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "questionary", specifier = ">=2.0" }, { name = "rich", specifier = ">=13.0" }, { name = "sqlalchemy", specifier = ">=2.0" }, { name = "typer", specifier = ">=0.24" }, @@ -1115,6 +1117,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/5e/9b994b5de36d6aa5caaf09a018d8fe4820db46e4da577c2fd7a1e176b56c/prek-0.3.1-py3-none-win_arm64.whl", hash = "sha256:cfa58365eb36753cff684dc3b00196c1163bb135fe72c6a1c6ebb1a179f5dbdf", size = 4021714, upload-time = "2026-01-31T13:25:34.993Z" }, ] +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + [[package]] name = "protobuf" version = "6.33.2" @@ -1368,6 +1382,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, ] +[[package]] +name = "questionary" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/45/eafb0bba0f9988f6a2520f9ca2df2c82ddfa8d67c95d6625452e97b204a5/questionary-2.1.1.tar.gz", hash = "sha256:3d7e980292bb0107abaa79c68dd3eee3c561b83a0f89ae482860b181c8bd412d", size = 25845, upload-time = "2025-08-28T19:00:20.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/26/1062c7ec1b053db9e499b4d2d5bc231743201b74051c973dadeac80a8f43/questionary-2.1.1-py3-none-any.whl", hash = "sha256:a51af13f345f1cdea62347589fbb6df3b290306ab8930713bfae4d475a7d4a59", size = 36753, upload-time = "2025-08-28T19:00:19.56Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -1686,6 +1712,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] +[[package]] +name = "wcwidth" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, +] + [[package]] name = "zipp" version = "3.23.0" From 32824d41c3f6076442695130d39550a2b9055d20 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 10:09:44 +0200 Subject: [PATCH 13/21] rename to .measure() --- docs/concepts/materializers.md | 2 +- docs/concepts/metrics.md | 20 ++-- docs/getting-started/quickstart.md | 12 +-- docs/index.md | 2 +- docs/plugins/conveyor.md | 4 +- docs/plugins/overview.md | 2 +- plugins/checkup-conveyor/README.md | 2 +- .../src/checkup_conveyor/conveyor_metric.py | 20 ++-- plugins/checkup-dbt/README.md | 2 +- .../src/checkup_dbt/metrics/base.py | 4 +- .../metrics/quality/flagged_packages.py | 6 +- .../metrics/quality/naming_convention.py | 2 +- .../metrics/quality/profile_host.py | 6 +- .../metrics/quality/supported_version.py | 2 +- .../checkup_dbt/metrics/quality/version.py | 2 +- .../metrics/test/column_test_coverage.py | 2 +- .../metrics/test/tested_columns.py | 2 +- plugins/checkup-git/README.md | 2 +- .../checkup-git/src/checkup_git/metrics.py | 12 +-- .../src/checkup_python/metrics/version.py | 2 +- .../checkup_python/metrics/version_check.py | 2 +- src/checkup/metric.py | 3 +- tests/fixtures.py | 44 ++++----- tests/test_executors.py | 14 +-- tests/test_hub.py | 6 +- tests/test_hub_execution.py | 8 +- tests/test_hub_validation.py | 2 +- tests/test_materializers.py | 94 +++++++++---------- 28 files changed, 140 insertions(+), 141 deletions(-) diff --git a/docs/concepts/materializers.md b/docs/concepts/materializers.md index bfd72ee..8fc27ee 100644 --- a/docs/concepts/materializers.md +++ b/docs/concepts/materializers.md @@ -153,7 +153,7 @@ Materializers group output by metric tags: ```python class MyMetric(Metric): def calculate(self, context, measurements): - return self.measurement( + return self.measure( value=42, tags={"domain": "analytics", "project": "dashboard"} ) diff --git a/docs/concepts/metrics.md b/docs/concepts/metrics.md index 38f389b..d9c68b7 100644 --- a/docs/concepts/metrics.md +++ b/docs/concepts/metrics.md @@ -19,7 +19,7 @@ class MyMetric(Metric): def calculate(self, context: Context, measurements: dict) -> Measurement: # Your calculation logic here - return self.measurement(value=42, diagnostic="Additional information") + return self.measure(value=42, diagnostic="Additional information") ``` ## Required Attributes @@ -45,10 +45,10 @@ The `calculate` method returns a `Measurement` object containing: | `diagnostic` | `str` | Additional diagnostic information | | `tags` | `dict` | Key-value pairs for grouping/filtering | -Use `self.measurement()` to create a `Measurement` with the metric's metadata pre-filled: +Use `self.measure()` to create a `Measurement` with the metric's metadata pre-filled: ```python -return self.measurement(value=42, diagnostic="Explanation", tags={"key": "value"}) +return self.measure(value=42, diagnostic="Explanation", tags={"key": "value"}) ``` ## The Calculate Method @@ -68,7 +68,7 @@ def calculate(self, context: Context, measurements: dict) -> Measurement: other_value = measurements[SomeOtherMetric].value # Return the result - return self.measurement(value=computed_value, diagnostic="Explanation of result") + return self.measure(value=computed_value, diagnostic="Explanation of result") ``` ## Dependencies @@ -82,7 +82,7 @@ class BaseMetric(Metric): unit = "count" def calculate(self, context: Context, measurements: dict) -> Measurement: - return self.measurement(value=100) + return self.measure(value=100) class DerivedMetric(Metric): @@ -96,7 +96,7 @@ class DerivedMetric(Metric): def calculate(self, context: Context, measurements: dict) -> Measurement: base_value = measurements[BaseMetric].value - return self.measurement(value=base_value * 0.5) + return self.measure(value=base_value * 0.5) ``` ## Providers @@ -124,7 +124,7 @@ class MyMetric(Metric): def calculate(self, context: Context, measurements: dict) -> Measurement: # Access provider data under its namespace data = context["my_data"] - return self.measurement(value=len(data.get("items", []))) + return self.measure(value=len(data.get("items", []))) ``` ## Executor Types @@ -167,7 +167,7 @@ class TaggedMetric(Metric): unit = "count" def calculate(self, context: Context, measurements: dict) -> Measurement: - return self.measurement( + return self.measure( value=42, tags={"domain": "data-platform", "project": "analytics"} ) @@ -213,7 +213,7 @@ class CodeCoverageMetric(Metric): coverage_data = context.get("coverage", {}) if not coverage_data: - return self.measurement(value=None, diagnostic="No coverage data available") + return self.measure(value=None, diagnostic="No coverage data available") total_lines = coverage_data.get("total_lines", 0) covered_lines = coverage_data.get("covered_lines", 0) @@ -225,7 +225,7 @@ class CodeCoverageMetric(Metric): value = 0 diagnostic = "No lines to cover" - return self.measurement( + return self.measure( value=value, diagnostic=diagnostic, tags={"project": coverage_data.get("project_name", "unknown")} diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index f23e9a3..2d89aa8 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -19,7 +19,7 @@ class FileCountMetric(Metric): def calculate(self, context: Context, measurements: dict) -> Measurement: # Access data from context and calculate your metric files = context.get("files", []) - return self.measurement( + return self.measure( value=len(files), diagnostic=f"Found {len(files)} files" ) @@ -29,7 +29,7 @@ Every metric must: 1. Define `name`, `description`, and `unit` class attributes 2. Implement the `calculate()` method -3. Return a `Measurement` using `self.measurement(value=..., diagnostic=...)` +3. Return a `Measurement` using `self.measure(value=..., diagnostic=...)` ## Running Metrics with CheckHub @@ -88,7 +88,7 @@ class FileCountMetric(Metric): def calculate(self, context: Context, measurements: dict) -> Measurement: # Access provider data under its namespace files = context["files"]["file_list"] - return self.measurement(value=len(files)) + return self.measure(value=len(files)) ``` Run with providers: @@ -119,7 +119,7 @@ class TotalLinesMetric(Metric): def calculate(self, context: Context, measurements: dict) -> Measurement: # Count lines in all files - return self.measurement(value=1000) # simplified + return self.measure(value=1000) # simplified class AverageLinesPerFileMetric(Metric): @@ -136,9 +136,9 @@ class AverageLinesPerFileMetric(Metric): total_lines = measurements[TotalLinesMetric].value if file_count > 0: - return self.measurement(value=total_lines / file_count) + return self.measure(value=total_lines / file_count) else: - return self.measurement(value=0, diagnostic="No files found") + return self.measure(value=0, diagnostic="No files found") ``` CheckUp automatically resolves dependencies and calculates metrics in the correct order. diff --git a/docs/index.md b/docs/index.md index 33732c1..baef1f6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,7 +26,7 @@ class SimpleMetric(Metric): unit = "count" def calculate(self, context: Context, measurements: dict) -> Measurement: - return self.measurement(value=42, diagnostic="Calculated successfully") + return self.measure(value=42, diagnostic="Calculated successfully") # Run the metric and output to console diff --git a/docs/plugins/conveyor.md b/docs/plugins/conveyor.md index 52a2419..79d2c5f 100644 --- a/docs/plugins/conveyor.md +++ b/docs/plugins/conveyor.md @@ -200,10 +200,10 @@ class SafeMetric(Metric): def calculate(self, context, measurements): conveyor = context.get("conveyor", {}) if conveyor.get("error"): - return self.measurement( + return self.measure( value=None, diagnostic=f"API Error: {conveyor['error']}" ) # Normal calculation... - return self.measurement(value=computed_value) + return self.measure(value=computed_value) ``` diff --git a/docs/plugins/overview.md b/docs/plugins/overview.md index 6a7b04f..90d94a6 100644 --- a/docs/plugins/overview.md +++ b/docs/plugins/overview.md @@ -130,7 +130,7 @@ class MyServiceMetric(Metric): def calculate(self, context: Context, measurements: dict) -> Measurement: data = context["myservice"]["data"] - return self.measurement(value=data.get("status", "unknown")) + return self.measure(value=data.get("status", "unknown")) ``` 5. Export in `__init__.py`: diff --git a/plugins/checkup-conveyor/README.md b/plugins/checkup-conveyor/README.md index bd0d58b..6858fd4 100644 --- a/plugins/checkup-conveyor/README.md +++ b/plugins/checkup-conveyor/README.md @@ -58,5 +58,5 @@ class MyConveyorMetric(ConveyorMetric): # Use api_client to fetch data from Conveyor value = ... - return self.measurement(value=value) + return self.measure(value=value) ``` diff --git a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py index 0eb96b0..eecbf66 100644 --- a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py +++ b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py @@ -20,7 +20,7 @@ def calculate( ) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: - return self.measurement(value=None) + return self.measure(value=None) r = requests.get( f"{self.base_url}/projects/{proj_id}/deployments", headers=self.get_conveyor_api_headers(context), @@ -28,8 +28,8 @@ def calculate( deployments = r.get("deployment", []) if not deployments: logger.warning("No deployments found for project %s", proj_id) - return self.measurement(value=None) - return self.measurement( + return self.measure(value=None) + return self.measure( value=deployments[0]["deployedOn"], diagnostic="Deploy the project again to update this value.", ) @@ -45,7 +45,7 @@ def calculate( ) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: - return self.measurement(value=None) + return self.measure(value=None) r = requests.get( f"{self.base_url}/projects/{proj_id}/builds", headers=self.get_conveyor_api_headers(context), @@ -53,12 +53,12 @@ def calculate( builds = r.get("builds", []) if not builds: logger.warning("No builds found for project %s", proj_id) - return self.measurement(value=None) + return self.measure(value=None) is_dirty = builds[0]["gitHash"].endswith(".dirty") diagnostic = ( "Commit changes to git, and deploy the project again." if is_dirty else "" ) - return self.measurement(value=is_dirty, diagnostic=diagnostic) + return self.measure(value=is_dirty, diagnostic=diagnostic) class ConveyorLastRunStatus(ConveyorMetric): @@ -71,10 +71,10 @@ def calculate( ) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: - return self.measurement(value=None) + return self.measure(value=None) env_id = self.get_environment_id(context) if env_id is None: - return self.measurement(value=None) + return self.measure(value=None) r = requests.get( f"{self.base_url}/environments/{env_id}/application_runs", params={ @@ -91,5 +91,5 @@ def calculate( proj_id, env_id, ) - return self.measurement(value=None) - return self.measurement(value=runs[0]["phase"]) + return self.measure(value=None) + return self.measure(value=runs[0]["phase"]) diff --git a/plugins/checkup-dbt/README.md b/plugins/checkup-dbt/README.md index 68f4653..7024d74 100644 --- a/plugins/checkup-dbt/README.md +++ b/plugins/checkup-dbt/README.md @@ -172,5 +172,5 @@ class MyCustomDbtMetric(DbtMetric): def calculate(self, context, measurements): manifest = self.get_manifest(context) - return self.measurement(value=len(manifest.nodes)) + return self.measure(value=len(manifest.nodes)) ``` diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py index 2a50793..b3fc40a 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py @@ -101,7 +101,7 @@ def calculate( value = query.count() logger.info(cls.log_message.format(value=value)) - return self.measurement(value=value) + return self.measure(value=value) class DbtDiagnosticMetric(DbtMetric): @@ -152,4 +152,4 @@ def calculate( else: diagnostic = f"{cls.diagnostic_prefix}: {', '.join(names)}" logger.info(cls.log_message.format(value=value)) - return self.measurement(value=value, diagnostic=diagnostic) + return self.measure(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py index 79532a0..44d7b2a 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py @@ -39,13 +39,13 @@ def calculate( if not packages_path.exists(): logger.warning(f"packages.yml not found at {packages_path}") - return self.measurement(value=0, diagnostic="packages.yml not found") + return self.measure(value=0, diagnostic="packages.yml not found") with open(packages_path) as f: packages_data = yaml.safe_load(f) if not packages_data or "packages" not in packages_data: - return self.measurement( + return self.measure( value=0, diagnostic="No packages defined in packages.yml" ) @@ -60,4 +60,4 @@ def calculate( value = len(flagged) diagnostic = f"Flagged packages: {', '.join(flagged)}" if flagged else "" logger.info(f"Found {value} flagged packages") - return self.measurement(value=value, diagnostic=diagnostic) + return self.measure(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py index 4a4402c..4704495 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py @@ -56,4 +56,4 @@ def calculate( if non_adhering_models: diagnostic = f"Models not adhering to naming convention: {', '.join(sorted(non_adhering_models))}" logger.info(f"Found {value} models not adhering to naming convention") - return self.measurement(value=value, diagnostic=diagnostic) + return self.measure(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py index 9a5976e..ac9c395 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py @@ -42,20 +42,20 @@ def calculate( if not profiles_path.exists(): logger.warning(f"profiles.yml not found at {profiles_path}") - return self.measurement(value=None, diagnostic="profiles.yml not found") + return self.measure(value=None, diagnostic="profiles.yml not found") with open(profiles_path) as f: profiles = yaml.safe_load(f) if not profiles: - return self.measurement(value=None, diagnostic="profiles.yml is empty") + return self.measure(value=None, diagnostic="profiles.yml is empty") host = self._find_host(profiles) if host: diagnostic = f"Host: {host}" else: diagnostic = f"No host found for target '{self.target}'" - return self.measurement(value=host, diagnostic=diagnostic) + return self.measure(value=host, diagnostic=diagnostic) def _find_host(self, profiles: dict) -> str | None: """Find host in profiles matching the configuration.""" diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py index 0f1f5a5..38b72e6 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py @@ -44,4 +44,4 @@ def calculate( f"Please upgrade dbt to version {self.min_version} or later." ) logger.info(f"dbt version {version} supported: {bool(value)}") - return self.measurement(value=value, diagnostic=diagnostic) + return self.measure(value=value, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py index 4b16458..1e4b473 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py @@ -24,4 +24,4 @@ def calculate( ) -> Measurement: manifest = self.get_manifest(context) value = manifest.metadata.dbt_version - return self.measurement(value=value, diagnostic=f"dbt version: {value}") + return self.measure(value=value, diagnostic=f"dbt version: {value}") diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py index 905148f..006b29b 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py @@ -38,4 +38,4 @@ def calculate( value = 0 logger.info(f"Column test coverage: {value}%") - return self.measurement(value=value) + return self.measure(value=value) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py index e221ac1..99819fb 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py @@ -45,4 +45,4 @@ def calculate( value = len(all_columns & tested_columns) logger.info(f"Found {value} tested columns") - return self.measurement(value=value) + return self.measure(value=value) diff --git a/plugins/checkup-git/README.md b/plugins/checkup-git/README.md index 754ac40..d1a9cc4 100644 --- a/plugins/checkup-git/README.md +++ b/plugins/checkup-git/README.md @@ -77,5 +77,5 @@ class MyCustomGitMetric(GitMetric): git_context = self.get_context(context) tracked_files = git_context.get("git_tracked_files", []) python_files = [f for f in tracked_files if f.endswith(".py")] - return self.measurement(value=len(python_files)) + return self.measure(value=len(python_files)) ``` diff --git a/plugins/checkup-git/src/checkup_git/metrics.py b/plugins/checkup-git/src/checkup_git/metrics.py index f8ac00c..bab58a1 100644 --- a/plugins/checkup-git/src/checkup_git/metrics.py +++ b/plugins/checkup-git/src/checkup_git/metrics.py @@ -36,11 +36,11 @@ def calculate( last_commit_date = git_context.get("git_last_commit_date") if not isinstance(last_commit_date, datetime): - return self.measurement(value=None, diagnostic="No commits found") + return self.measure(value=None, diagnostic="No commits found") now = datetime.now(UTC) delta = now - last_commit_date - return self.measurement( + return self.measure( value=delta.days, diagnostic=f"Last commit: {last_commit_date.strftime('%Y-%m-%d')}", ) @@ -74,19 +74,19 @@ def calculate( tracked_files = git_context.get("git_tracked_files", []) if not isinstance(tracked_files, list): - return self.measurement(value=0, diagnostic="No git repository found") + return self.measure(value=0, diagnostic="No git repository found") if self.pattern != "*": matched_files = [f for f in tracked_files if fnmatch(f, self.pattern)] if matched_files: - return self.measurement( + return self.measure( value=len(matched_files), diagnostic=f"Matched files: {', '.join(matched_files)}", ) else: - return self.measurement( + return self.measure( value=len(matched_files), diagnostic=f"No files matching pattern: {self.pattern}", ) else: - return self.measurement(value=len(tracked_files)) + return self.measure(value=len(tracked_files)) diff --git a/plugins/checkup-python/src/checkup_python/metrics/version.py b/plugins/checkup-python/src/checkup_python/metrics/version.py index f475c8a..db9f031 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version.py @@ -37,7 +37,7 @@ def calculate( or self._get_runtime_version() ) - return self.measurement(value=version) + return self.measure(value=version) def _read_python_version_file(self, path: Path) -> str | None: """ diff --git a/plugins/checkup-python/src/checkup_python/metrics/version_check.py b/plugins/checkup-python/src/checkup_python/metrics/version_check.py index a946dd8..11981ae 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version_check.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version_check.py @@ -35,4 +35,4 @@ def calculate( max_ver = parse_semantic_version(self.max_version) value = min_ver <= actual <= max_ver - return self.measurement(value=value) + return self.measure(value=value) diff --git a/src/checkup/metric.py b/src/checkup/metric.py index 987a6ec..d36b9a0 100644 --- a/src/checkup/metric.py +++ b/src/checkup/metric.py @@ -58,7 +58,7 @@ def calculate( """ pass - def measurement( + def measure( self, value: Any = None, tags: dict | None = None, @@ -77,7 +77,6 @@ def measurement( Returns: Measurement instance """ - return Measurement( metric=self, value=value, diff --git a/tests/fixtures.py b/tests/fixtures.py index fe8257f..0b5fa22 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -18,7 +18,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: """Set value to expected_value.""" - return self.measurement( + return self.measure( value=self.expected_value, diagnostic=f"Dummy metric calculated with expected_value={self.expected_value}", ) @@ -42,7 +42,7 @@ def calculate( """Double the DummyMetric value.""" base_value = measurements[DummyMetric].value value = base_value * 2 - return self.measurement( + return self.measure( value=value, diagnostic=f"Doubled DummyMetric value from {base_value} to {value}", ) @@ -63,7 +63,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: value = measurements[DependentDummyMetric].value + 10 - return self.measurement( + return self.measure( value=value, diagnostic=f"Added 10 to DependentDummyMetric value: {value}" ) @@ -84,7 +84,7 @@ def calculate( ) -> Measurement: level2_value = measurements[Level2Metric].value value = level2_value**2 - return self.measurement( + return self.measure( value=value, diagnostic=f"Squared Level2Metric value: {level2_value}^2 = {value}", ) @@ -104,7 +104,7 @@ def depends_on(cls) -> list[type[Metric]]: def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement(value=1, diagnostic="CyclicMetricA calculated") + return self.measure(value=1, diagnostic="CyclicMetricA calculated") class CyclicMetricB(Metric): @@ -121,7 +121,7 @@ def depends_on(cls) -> list[type[Metric]]: def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement(value=1, diagnostic="CyclicMetricB calculated") + return self.measure(value=1, diagnostic="CyclicMetricB calculated") class RootA(Metric): @@ -135,7 +135,7 @@ class RootA(Metric): def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement( + return self.measure( value=self.base_value, diagnostic=f"RootA calculated with base_value={self.base_value}", ) @@ -152,7 +152,7 @@ class RootB(Metric): def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement( + return self.measure( value=self.base_value, diagnostic=f"RootB calculated with base_value={self.base_value}", ) @@ -169,7 +169,7 @@ class RootC(Metric): def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement( + return self.measure( value=self.base_value, diagnostic=f"RootC calculated with base_value={self.base_value}", ) @@ -192,7 +192,7 @@ def calculate( root_a_val = measurements[RootA].value root_b_val = measurements[RootB].value value = root_a_val + root_b_val - return self.measurement( + return self.measure( value=value, diagnostic=f"Sum of RootA ({root_a_val}) and RootB ({root_b_val}) = {value}", ) @@ -214,7 +214,7 @@ def calculate( ) -> Measurement: root_b_val = measurements[RootB].value value = root_b_val * 3 - return self.measurement( + return self.measure( value=value, diagnostic=f"Tripled RootB value: {root_b_val} * 3 = {value}" ) @@ -235,7 +235,7 @@ def calculate( ) -> Measurement: root_c_val = measurements[RootC].value value = root_c_val**2 - return self.measurement( + return self.measure( value=value, diagnostic=f"Squared RootC value: {root_c_val}^2 = {value}" ) @@ -256,7 +256,7 @@ def calculate( ) -> Measurement: shared_ab_val = measurements[SharedAB].value value = shared_ab_val + 5 - return self.measurement( + return self.measure( value=value, diagnostic=f"Added 5 to SharedAB value: {shared_ab_val} + 5 = {value}", ) @@ -278,7 +278,7 @@ def calculate( ) -> Measurement: branch_b_val = measurements[BranchB].value value = branch_b_val * 2 - return self.measurement( + return self.measure( value=value, diagnostic=f"Doubled BranchB value: {branch_b_val} * 2 = {value}", ) @@ -301,7 +301,7 @@ def calculate( mid_shared_val = measurements[MidShared].value mid_branch_val = measurements[MidBranch].value value = mid_shared_val * mid_branch_val - return self.measurement( + return self.measure( value=value, diagnostic=f"Product of MidShared ({mid_shared_val}) and MidBranch ({mid_branch_val}) = {value}", ) @@ -334,7 +334,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: value = context[DummyProvider.name]["data"] - return self.measurement( + return self.measure( value=value, diagnostic=f"Retrieved dummy_data from context: {value}" ) @@ -351,7 +351,7 @@ def calculate( ) -> Measurement: if context.get("should_fail"): raise ValueError("Intentional failure") - return self.measurement(value=1, diagnostic="Metric calculated successfully") + return self.measure(value=1, diagnostic="Metric calculated successfully") class IntegrationProvider(Provider): @@ -382,7 +382,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: value = context[IntegrationProvider.name]["base_value"] - return self.measurement( + return self.measure( value=value, diagnostic=f"Retrieved base_value from integration provider: {value}", ) @@ -405,7 +405,7 @@ def calculate( ) -> Measurement: base_val = measurements[IntegrationBaseMetric].value value = base_val * self.multiplier - return self.measurement( + return self.measure( value=value, diagnostic=f"Multiplied base metric value: {base_val} * {self.multiplier} = {value}", ) @@ -440,7 +440,7 @@ def calculate( ) -> Measurement: path_len = context[PathLengthProvider.name]["length"] value = path_len * self.multiplier - return self.measurement( + return self.measure( value=value, diagnostic=f"Path length {path_len} * multiplier {self.multiplier} = {value}", ) @@ -459,7 +459,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: """Set value to expected_value.""" - return self.measurement( + return self.measure( value=self.expected_value, diagnostic=f"Other metric calculated with expected_value={self.expected_value}", ) @@ -478,7 +478,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: """Set value to expected_value.""" - return self.measurement( + return self.measure( value=self.expected_value, diagnostic=f"Indirect metric calculated with expected_value={self.expected_value}", ) diff --git a/tests/test_executors.py b/tests/test_executors.py index 46f7487..ac5798e 100644 --- a/tests/test_executors.py +++ b/tests/test_executors.py @@ -18,7 +18,7 @@ class ThreadMetric(Metric): def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement(value=10, diagnostic="Calculated in thread") + return self.measure(value=10, diagnostic="Calculated in thread") class ProcessMetric(Metric): @@ -32,7 +32,7 @@ class ProcessMetric(Metric): def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement(value=20, diagnostic="Calculated in process") + return self.measure(value=20, diagnostic="Calculated in process") class AsyncMetric(Metric): @@ -46,7 +46,7 @@ class AsyncMetric(Metric): def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement(value=30, diagnostic="Calculated with asyncio") + return self.measure(value=30, diagnostic="Calculated with asyncio") class AsyncMetricWithAsyncCalculate(Metric): @@ -63,7 +63,7 @@ async def calculate( import asyncio await asyncio.sleep(0.001) # Small async operation - return self.measurement(value=40, diagnostic="Calculated with native async") + return self.measure(value=40, diagnostic="Calculated with native async") class DependentThreadMetric(Metric): @@ -83,7 +83,7 @@ def calculate( ) -> Measurement: base_value = measurements[ThreadMetric].value value = base_value * 2 - return self.measurement( + return self.measure( value=value, diagnostic=f"Doubled thread metric: {base_value} -> {value}" ) @@ -105,7 +105,7 @@ def calculate( ) -> Measurement: base_value = measurements[ThreadMetric].value value = base_value * 3 - return self.measurement( + return self.measure( value=value, diagnostic=f"Tripled thread metric: {base_value} -> {value}" ) @@ -127,7 +127,7 @@ def calculate( ) -> Measurement: base_value = measurements[ProcessMetric].value value = base_value + 5 - return self.measurement( + return self.measure( value=value, diagnostic=f"Added 5 to process metric: {base_value} -> {value}", ) diff --git a/tests/test_hub.py b/tests/test_hub.py index cd86501..54cb9ed 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -31,7 +31,7 @@ def test_checkhub_with_metrics(): def test_measurement_result_creation(): """Test creating a MeasurementResult.""" metric = DummyMetric() - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) result = MeasurementResult(measurements=[measurement]) assert len(result.measurements) == 1 @@ -43,7 +43,7 @@ def test_measurement_result_with_errors(): from checkup.providers.tags import TagProvider metric = DummyMetric() - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) provider = TagProvider(path="/bad/path") errors = [([provider], ValueError("Path not found"))] @@ -58,7 +58,7 @@ def test_measurement_result_with_errors(): def test_measurement_result_errors_default_empty(): """Test MeasurementResult.errors defaults to empty list.""" metric = DummyMetric() - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) result = MeasurementResult(measurements=[measurement]) assert result.errors == [] diff --git a/tests/test_hub_execution.py b/tests/test_hub_execution.py index 3e131ca..2d089fb 100644 --- a/tests/test_hub_execution.py +++ b/tests/test_hub_execution.py @@ -36,7 +36,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: value = context[DataProvider.name]["value"] - return self.measurement(value=value) + return self.measure(value=value) class OtherProvider(Provider): @@ -71,7 +71,7 @@ def providers(cls) -> list[type[Provider]]: def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement(value=999) + return self.measure(value=999) class DependsOnFailingMetric(Metric): @@ -93,7 +93,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: base_val = measurements[FailingProviderMetric].value - return self.measurement(value=base_val * 2) + return self.measure(value=base_val * 2) class OtherMetric(Metric): @@ -111,7 +111,7 @@ def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: value = context[OtherProvider.name]["other_value"] - return self.measurement(value=value) + return self.measure(value=value) class TestHubExecution: diff --git a/tests/test_hub_validation.py b/tests/test_hub_validation.py index 82d4a9b..9614ec4 100644 --- a/tests/test_hub_validation.py +++ b/tests/test_hub_validation.py @@ -40,7 +40,7 @@ def providers(cls) -> list[type[Provider]]: def calculate( self, context: Context, measurements: dict[type[Metric], Measurement] ) -> Measurement: - return self.measurement(value=1) + return self.measure(value=1) class TestProviderValidation: diff --git a/tests/test_materializers.py b/tests/test_materializers.py index 8a3a7b8..48d4484 100644 --- a/tests/test_materializers.py +++ b/tests/test_materializers.py @@ -26,7 +26,7 @@ def test_materializer_is_abstract(): def test_console_materializer(): """Test console output materializer.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) # Capture stdout captured_output = StringIO() @@ -46,7 +46,7 @@ def test_console_materializer(): def test_csv_materializer(tmp_path): """Test CSV file materializer.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) output_file = tmp_path / "metrics.csv" materializer = CSVMaterializer(output_path=output_file) @@ -70,10 +70,10 @@ def test_csv_materializer_multiple_metrics(tmp_path): from fixtures import OtherDummyMetric metric1 = DummyMetric(expected_value=42) - measurement1 = metric1.measurement(value=42) + measurement1 = metric1.measure(value=42) metric2 = OtherDummyMetric(expected_value=100) - measurement2 = metric2.measurement(value=100) + measurement2 = metric2.measure(value=100) output_file = tmp_path / "metrics.csv" materializer = CSVMaterializer(output_path=output_file) @@ -92,10 +92,10 @@ def test_materializer_filters_indirect_by_default(): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_measurement = direct_metric.measurement(value=42) + direct_measurement = direct_metric.measure(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_measurement = indirect_metric.measurement(value=100) + indirect_measurement = indirect_metric.measure(value=100) # Capture stdout captured_output = StringIO() @@ -117,10 +117,10 @@ def test_materializer_includes_indirect_when_configured(): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_measurement = direct_metric.measurement(value=42) + direct_measurement = direct_metric.measure(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_measurement = indirect_metric.measurement(value=100) + indirect_measurement = indirect_metric.measure(value=100) # Capture stdout captured_output = StringIO() @@ -143,10 +143,10 @@ def test_csv_materializer_filters_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_measurement = direct_metric.measurement(value=42) + direct_measurement = direct_metric.measure(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_measurement = indirect_metric.measurement(value=100) + indirect_measurement = indirect_metric.measure(value=100) output_file = tmp_path / "metrics.csv" @@ -168,10 +168,10 @@ def test_csv_materializer_includes_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_measurement = direct_metric.measurement(value=42) + direct_measurement = direct_metric.measure(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_measurement = indirect_metric.measurement(value=100) + indirect_measurement = indirect_metric.measure(value=100) output_file = tmp_path / "metrics.csv" @@ -191,17 +191,17 @@ def test_html_materializer(tmp_path): """Test HTML materializer with hierarchical grouping.""" # Create measurements with tags metric1 = DummyMetric(expected_value=42) - measurement1 = metric1.measurement( + measurement1 = metric1.measure( value=42, tags={"domain": "Analytics", "project": "Project A"} ) metric2 = DummyMetric(expected_value=100) - measurement2 = metric2.measurement( + measurement2 = metric2.measure( value=100, tags={"domain": "Analytics", "project": "Project B"} ) metric3 = DummyMetric(expected_value=75) - measurement3 = metric3.measurement( + measurement3 = metric3.measure( value=75, tags={"domain": "Engineering", "project": "Project C"} ) @@ -255,14 +255,14 @@ def test_html_materializer_with_diagnostics(tmp_path): """Test HTML materializer with diagnostic coloring.""" # Create measurements with different diagnostics metric1 = DummyMetric(expected_value=42) - measurement1 = metric1.measurement( + measurement1 = metric1.measure( value=42, diagnostic="✅ All good", tags={"domain": "Test", "project": "TestProject"}, ) metric2 = DummyMetric(expected_value=100) - measurement2 = metric2.measurement( + measurement2 = metric2.measure( value=100, diagnostic="⚠ Warning: something to check", tags={"domain": "Test", "project": "TestProject"}, @@ -289,7 +289,7 @@ def test_html_materializer_ungrouped_metrics(tmp_path): """Test HTML materializer with measurements missing tags.""" # Create measurement without tags metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42, tags={}) + measurement = metric.measure(value=42, tags={}) output_file = tmp_path / "metrics.html" materializer = HTMLMaterializer( @@ -309,12 +309,12 @@ def test_html_materializer_filters_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_measurement = direct_metric.measurement( + direct_measurement = direct_metric.measure( value=42, tags={"domain": "Test", "project": "TestProject"} ) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_measurement = indirect_metric.measurement( + indirect_measurement = indirect_metric.measure( value=100, tags={"domain": "Test", "project": "TestProject"} ) @@ -336,7 +336,7 @@ def test_html_materializer_filters_indirect(tmp_path): def test_html_materializer_escape_html(tmp_path): """Test that HTML special characters are escaped.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement( + measurement = metric.measure( value="", diagnostic="Test & verify ", tags={"domain": "Test & Dev", "project": "Project "}, @@ -366,28 +366,28 @@ def test_html_materializer_end_to_end(tmp_path): # Create measurements for Analytics domain metric1 = DummyMetric(expected_value=42) - measurement1 = metric1.measurement( + measurement1 = metric1.measure( value=42, diagnostic="✅ Good coverage", tags={"domain": "Analytics", "project": "Customer Insights"}, ) metric2 = OtherDummyMetric(expected_value=85) - measurement2 = metric2.measurement( + measurement2 = metric2.measure( value=85, diagnostic="", tags={"domain": "Analytics", "project": "Customer Insights"}, ) metric3 = DummyMetric(expected_value=100) - measurement3 = metric3.measurement( + measurement3 = metric3.measure( value=100, diagnostic="✅ Excellent", tags={"domain": "Analytics", "project": "Sales Dashboard"}, ) metric4 = OtherDummyMetric(expected_value=60) - measurement4 = metric4.measurement( + measurement4 = metric4.measure( value=60, diagnostic="⚠ Below target", tags={"domain": "Analytics", "project": "Sales Dashboard"}, @@ -395,41 +395,41 @@ def test_html_materializer_end_to_end(tmp_path): # Create measurements for Engineering domain metric5 = DummyMetric(expected_value=95) - measurement5 = metric5.measurement( + measurement5 = metric5.measure( value=95, diagnostic="✅ Strong test coverage", tags={"domain": "Engineering", "project": "Core Platform"}, ) metric6 = OtherDummyMetric(expected_value=45) - measurement6 = metric6.measurement( + measurement6 = metric6.measure( value=45, diagnostic="❌ Critical - needs attention", tags={"domain": "Engineering", "project": "Core Platform"}, ) metric7 = DummyMetric(expected_value=78) - measurement7 = metric7.measurement( + measurement7 = metric7.measure( value=78, diagnostic="", tags={"domain": "Engineering", "project": "Mobile App"} ) # Create measurements for Data Science domain metric8 = DummyMetric(expected_value=92) - measurement8 = metric8.measurement( + measurement8 = metric8.measure( value=92, diagnostic="✅ Model accuracy within range", tags={"domain": "Data Science", "project": "ML Pipeline"}, ) metric9 = OtherDummyMetric(expected_value=88) - measurement9 = metric9.measurement( + measurement9 = metric9.measure( value=88, diagnostic="", tags={"domain": "Data Science", "project": "ML Pipeline"}, ) metric10 = DummyMetric(expected_value=55) - measurement10 = metric10.measurement( + measurement10 = metric10.measure( value=55, diagnostic="⚠ Training data quality concerns", tags={"domain": "Data Science", "project": "Recommendation Engine"}, @@ -437,7 +437,7 @@ def test_html_materializer_end_to_end(tmp_path): # Create some ungrouped measurements metric11 = DummyMetric(expected_value=70) - measurement11 = metric11.measurement(value=70, diagnostic="", tags={}) + measurement11 = metric11.measure(value=70, diagnostic="", tags={}) all_measurements = [ measurement1, @@ -500,7 +500,7 @@ def test_html_materializer_end_to_end(tmp_path): def test_sqlalchemy_materializer(tmp_path): """Test SQLAlchemy materializer writes measurements to database.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42, tags={"domain": "Analytics"}) + measurement = metric.measure(value=42, tags={"domain": "Analytics"}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -527,10 +527,10 @@ def test_sqlalchemy_materializer_multiple_metrics(tmp_path): from fixtures import OtherDummyMetric metric1 = DummyMetric(expected_value=42) - measurement1 = metric1.measurement(value=42) + measurement1 = metric1.measure(value=42) metric2 = OtherDummyMetric(expected_value=100) - measurement2 = metric2.measurement(value=100) + measurement2 = metric2.measure(value=100) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -550,7 +550,7 @@ def test_sqlalchemy_materializer_multiple_metrics(tmp_path): def test_sqlalchemy_materializer_appends_rows(tmp_path): """Test that successive materializations append rows.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) db_path = tmp_path / "metrics.db" url = f"sqlite:///{db_path}" @@ -570,7 +570,7 @@ def test_sqlalchemy_materializer_appends_rows(tmp_path): def test_sqlalchemy_materializer_custom_table_name(tmp_path): """Test SQLAlchemy materializer with custom table name.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -592,10 +592,10 @@ def test_sqlalchemy_materializer_filters_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_measurement = direct_metric.measurement(value=42) + direct_measurement = direct_metric.measure(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_measurement = indirect_metric.measurement(value=100) + indirect_measurement = indirect_metric.measure(value=100) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -616,10 +616,10 @@ def test_sqlalchemy_materializer_includes_indirect(tmp_path): from fixtures import IndirectDummyMetric direct_metric = DummyMetric(expected_value=42) - direct_measurement = direct_metric.measurement(value=42) + direct_measurement = direct_metric.measure(value=42) indirect_metric = IndirectDummyMetric(expected_value=100) - indirect_measurement = indirect_metric.measurement(value=100) + indirect_measurement = indirect_metric.measure(value=100) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -640,7 +640,7 @@ def test_sqlalchemy_materializer_includes_indirect(tmp_path): def test_sqlalchemy_materializer_none_value(tmp_path): """Test SQLAlchemy materializer handles None values.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=None, tags={}) + measurement = metric.measure(value=None, tags={}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -702,17 +702,17 @@ def test_sqlalchemy_materializer_expand_tags(tmp_path): from fixtures import IndirectDummyMetric, OtherDummyMetric metric1 = DummyMetric(expected_value=42) - measurement1 = metric1.measurement( + measurement1 = metric1.measure( value=42, tags={"domain": "Analytics", "env": "prod"} ) metric2 = OtherDummyMetric(expected_value=100) - measurement2 = metric2.measurement( + measurement2 = metric2.measure( value=100, tags={"domain": "Engineering", "team": "platform"} ) metric3 = IndirectDummyMetric(expected_value=50) - measurement3 = metric3.measurement(value=50, tags={}) + measurement3 = metric3.measure(value=50, tags={}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -766,7 +766,7 @@ def test_sqlalchemy_materializer_expand_tags(tmp_path): def test_sqlalchemy_materializer_expand_tags_no_tags(tmp_path): """Test expand_tags handles measurements with no tags.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42, tags={}) + measurement = metric.measure(value=42, tags={}) db_path = tmp_path / "metrics.db" materializer = SQLAlchemyMaterializer( @@ -796,7 +796,7 @@ def test_sqlalchemy_materializer_batch_size(tmp_path): measurements = [] for i in range(25): metric = DummyMetric(expected_value=i) - measurement = metric.measurement(value=i) + measurement = metric.measure(value=i) measurements.append(measurement) db_path = tmp_path / "metrics.db" From 29b9f9095eff23b92bb2354b52306bff7f622d8f Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 10:12:57 +0200 Subject: [PATCH 14/21] rebase --- tests/test_materializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_materializers.py b/tests/test_materializers.py index 34459e5..65fe3ac 100644 --- a/tests/test_materializers.py +++ b/tests/test_materializers.py @@ -46,7 +46,7 @@ def test_console_materializer(): def test_console_materializer_no_grouping(): """Test console materializer without grouping.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42) + measurement = metric.measure(value=42) captured_output = StringIO() sys.stdout = captured_output @@ -64,7 +64,7 @@ def test_console_materializer_no_grouping(): def test_console_materializer_single_grouping(): """Test console materializer with single-level grouping.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement(value=42, tags={"domain": "Analytics"}) + measurement = metric.measure(value=42, tags={"domain": "Analytics"}) captured_output = StringIO() sys.stdout = captured_output @@ -83,7 +83,7 @@ def test_console_materializer_single_grouping(): def test_console_materializer_three_level_grouping(): """Test console materializer with three-level grouping.""" metric = DummyMetric(expected_value=42) - measurement = metric.measurement( + measurement = metric.measure( value=42, tags={"domain": "Analytics", "project": "Core", "env": "prod"} ) From 53d816eb5560edfc824a6456425be4dd6fe69bd3 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 11:42:28 +0200 Subject: [PATCH 15/21] remove check --- src/checkup/cli/__init__.py | 3 +- src/checkup/cli/commands/__init__.py | 2 -- src/checkup/cli/commands/check.py | 48 ---------------------------- src/checkup/cli/commands/run.py | 2 +- src/checkup/cli/config_wizard.py | 2 +- 5 files changed, 3 insertions(+), 54 deletions(-) delete mode 100644 src/checkup/cli/commands/check.py diff --git a/src/checkup/cli/__init__.py b/src/checkup/cli/__init__.py index 15ceaa4..c23f005 100644 --- a/src/checkup/cli/__init__.py +++ b/src/checkup/cli/__init__.py @@ -4,7 +4,7 @@ import typer -from checkup.cli.commands import check, config, init, run, schema +from checkup.cli.commands import config, init, run, schema app = typer.Typer( name="checkup", @@ -12,7 +12,6 @@ no_args_is_help=True, ) -app.command()(check) app.command()(run) app.command()(init) app.command()(config) diff --git a/src/checkup/cli/commands/__init__.py b/src/checkup/cli/commands/__init__.py index 55a6a66..3e99bc9 100644 --- a/src/checkup/cli/commands/__init__.py +++ b/src/checkup/cli/commands/__init__.py @@ -2,14 +2,12 @@ CLI commands. """ -from checkup.cli.commands.check import check from checkup.cli.commands.config import config from checkup.cli.commands.init import init from checkup.cli.commands.run import run from checkup.cli.commands.schema import schema __all__ = [ - "check", "config", "init", "run", diff --git a/src/checkup/cli/commands/check.py b/src/checkup/cli/commands/check.py deleted file mode 100644 index 6039ec3..0000000 --- a/src/checkup/cli/commands/check.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Check command. Run metrics locally. -""" - -import logging -from pathlib import Path -from typing import Annotated - -import typer - -from checkup.cli.executor import execute_checkup -from checkup.cli.utils import apply_cli_overrides -from checkup.configuration import load_config - - -def check( - config: Annotated[ - Path | None, - typer.Option("--config", "-c", help="Path to config file"), - ] = None, - tag: Annotated[ - list[str] | None, - typer.Option("--tag", "-t", help="Set tags (key=value)"), - ] = None, - provider: Annotated[ - list[str] | None, - typer.Option("--provider", "-p", help="Set providers (name or name:key=value)"), - ] = None, - metric: Annotated[ - list[str] | None, - typer.Option("--metric", "-m", help="Set metrics (name or name:key=value)"), - ] = None, - verbose: Annotated[ - bool, - typer.Option("--verbose", "-v", help="Verbose output"), - ] = False, -) -> None: - """ - Run metrics and show results. - """ - - if verbose: - logging.basicConfig(level=logging.DEBUG) - - cfg = load_config(config_path=config) - cfg = apply_cli_overrides(cfg, tag, provider, metric) - - execute_checkup(cfg, materializer="console") diff --git a/src/checkup/cli/commands/run.py b/src/checkup/cli/commands/run.py index 1d1ca49..5fab572 100644 --- a/src/checkup/cli/commands/run.py +++ b/src/checkup/cli/commands/run.py @@ -38,7 +38,7 @@ def run( ] = None, dry_run: Annotated[ bool, - typer.Option("--dry-run", help="Don't materialize, just print (same as check)"), + typer.Option("--dry-run", help="Don't materialize, just print"), ] = False, verbose: Annotated[ bool, diff --git a/src/checkup/cli/config_wizard.py b/src/checkup/cli/config_wizard.py index 351200a..cced1b1 100644 --- a/src/checkup/cli/config_wizard.py +++ b/src/checkup/cli/config_wizard.py @@ -399,5 +399,5 @@ def _write_config(path: Path, config: dict) -> None: console.print("[green]Done![/green]", markup=True) console.print( - "Run [bold]checkup check[/bold] to test your configuration.", markup=True + "Run [bold]checkup run[/bold] to test your configuration.", markup=True ) From 78cb071ef1ce5da6e201c504f3b7444f54d6c188 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 12:56:55 +0200 Subject: [PATCH 16/21] update --- src/checkup/configuration/schema.py | 66 ++++++++++++++--------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/checkup/configuration/schema.py b/src/checkup/configuration/schema.py index 990b9d1..e9b2301 100644 --- a/src/checkup/configuration/schema.py +++ b/src/checkup/configuration/schema.py @@ -2,9 +2,10 @@ JSON Schema generation for checkup.yaml. """ +import inspect import json from pathlib import Path -from typing import Any +from typing import Any, get_origin, get_type_hints from checkup.registry import get_registry @@ -14,7 +15,7 @@ def _get_pydantic_schema(cls: type) -> dict | None: """ - Get JSON schema from a Pydantic model, filtering internal fields. + Get JSON schema from a Pydantic model. """ if not hasattr(cls, "model_json_schema"): @@ -23,20 +24,9 @@ def _get_pydantic_schema(cls: type) -> dict | None: try: schema = cls.model_json_schema() - # Remove internal fields for metrics - if "properties" in schema: - for field in ("value", "diagnostic", "tags"): - schema["properties"].pop(field, None) - if "required" in schema: - schema["required"] = [ - r - for r in schema["required"] - if r not in ("value", "diagnostic", "tags") - ] - - # Remove $defs if present (inline everything) - schema.pop("$defs", None) - schema.pop("title", None) + # Remove Pydantic metadata we don't need in the config schema + schema.pop("$defs", None) # Inline type definitions + schema.pop("title", None) # We use entry point names, not class names if not schema.get("properties"): return None @@ -46,14 +36,35 @@ def _get_pydantic_schema(cls: type) -> dict | None: return None +def _python_type_to_json_schema_type(hint: type) -> str: + """ + Map a Python type hint to a JSON Schema type. + """ + + origin = get_origin(hint) + if origin is not None: + # For Union types, just use string as fallback + return "string" + + if hint is str: + return "string" + if hint is int: + return "integer" + if hint is float: + return "number" + if hint is bool: + return "boolean" + if hint is Path or (isinstance(hint, type) and issubclass(hint, Path)): + return "string" + + return "string" + + def _get_provider_schema(cls: type) -> dict | None: """ Get JSON schema for a provider from its __init__ signature. """ - import inspect - from typing import get_type_hints - try: sig = inspect.signature(cls.__init__) hints = get_type_hints(cls.__init__) @@ -69,26 +80,11 @@ def _get_provider_schema(cls: type) -> dict | None: prop: dict[str, Any] = {} - # Get type if name in hints: - hint = hints[name] - type_name = getattr(hint, "__name__", str(hint)) - if "str" in type_name or "str" in str(hint): - prop["type"] = "string" - elif "int" in type_name: - prop["type"] = "integer" - elif "float" in type_name: - prop["type"] = "number" - elif "bool" in type_name: - prop["type"] = "boolean" - elif "Path" in str(hint): - prop["type"] = "string" - else: - prop["type"] = "string" + prop["type"] = _python_type_to_json_schema_type(hints[name]) else: prop["type"] = "string" - # Get default if param.default is not inspect.Parameter.empty: default = param.default # Convert Path to string for JSON From 78a782c6ac2286ef197ffd04276838e0b3011e2a Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 13:00:36 +0200 Subject: [PATCH 17/21] update --- src/checkup/cli/executor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/checkup/cli/executor.py b/src/checkup/cli/executor.py index 89424d2..c587c76 100644 --- a/src/checkup/cli/executor.py +++ b/src/checkup/cli/executor.py @@ -17,6 +17,7 @@ from checkup.materializers import Materializer from checkup.metric import Metric from checkup.provider import Provider + from checkup.registry.discovery import PluginRegistry logger = logging.getLogger(__name__) console = Console() @@ -145,7 +146,3 @@ def _resolve_materializer( except Exception as e: console.print(f"[red]Failed to instantiate materializer {mat_type}: {e}[/red]") return ConsoleMaterializer() - - -# Import for type hints -from checkup.registry.discovery import PluginRegistry # noqa: E402 From ac35f1248611f9a5c3e5926f9bf7ad02fb9d861e Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Tue, 21 Apr 2026 13:20:08 +0200 Subject: [PATCH 18/21] redundant tests --- tests/test_registry.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/tests/test_registry.py b/tests/test_registry.py index b57cb96..e4a84ea 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -152,36 +152,3 @@ def test_list_metric_names_without_loading(self): assert names == ["git_days_since_last_update"] mock_ep.load.assert_not_called() - - -class TestRegistryLoading: - def test_providers_property_loads_and_caches_entry_points(self): - registry = PluginRegistry() - - with patch("checkup.registry.discovery.entry_points") as mock_eps: - mock_ep = MagicMock() - mock_ep.name = "test_provider" - mock_ep.load.return_value = "LoadedProviderClass" - mock_eps.return_value = [mock_ep] - - providers1 = registry.providers - providers2 = registry.providers - - assert providers1 == {"test_provider": "LoadedProviderClass"} - assert providers1 is providers2 - assert mock_eps.call_count == 1 - - def test_clear_cache_allows_reload(self): - registry = PluginRegistry() - - with patch("checkup.registry.discovery.entry_points") as mock_eps: - mock_ep = MagicMock() - mock_ep.name = "test" - mock_ep.load.return_value = "Class" - mock_eps.return_value = [mock_ep] - - _ = registry.providers - registry.clear_cache() - _ = registry.providers - - assert mock_eps.call_count == 2 From 408149c1198a6bcdea9f7d5e74463ce87ccc9a01 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Thu, 23 Apr 2026 10:12:58 +0200 Subject: [PATCH 19/21] split wizard module --- src/checkup/cli/commands/schema.py | 3 +- src/checkup/cli/config_wizard.py | 403 ---------------------- src/checkup/cli/config_wizard/__init__.py | 8 + src/checkup/cli/config_wizard/_common.py | 133 +++++++ src/checkup/cli/config_wizard/create.py | 134 +++++++ src/checkup/cli/config_wizard/edit.py | 200 +++++++++++ src/checkup/configuration/io.py | 1 + 7 files changed, 478 insertions(+), 404 deletions(-) delete mode 100644 src/checkup/cli/config_wizard.py create mode 100644 src/checkup/cli/config_wizard/__init__.py create mode 100644 src/checkup/cli/config_wizard/_common.py create mode 100644 src/checkup/cli/config_wizard/create.py create mode 100644 src/checkup/cli/config_wizard/edit.py diff --git a/src/checkup/cli/commands/schema.py b/src/checkup/cli/commands/schema.py index 7fbe768..572698b 100644 --- a/src/checkup/cli/commands/schema.py +++ b/src/checkup/cli/commands/schema.py @@ -8,6 +8,7 @@ import typer from rich.console import Console +from checkup.configuration.io import SCHEMA_FILENAME from checkup.configuration.schema import write_schema console = Console() @@ -23,6 +24,6 @@ def schema( Generate JSON schema for checkup.yaml. """ - path = output or Path.cwd() / "checkup.schema.json" + path = output or Path.cwd() / SCHEMA_FILENAME write_schema(path) console.print(f"[green]Schema written to {path}[/green]") diff --git a/src/checkup/cli/config_wizard.py b/src/checkup/cli/config_wizard.py deleted file mode 100644 index cced1b1..0000000 --- a/src/checkup/cli/config_wizard.py +++ /dev/null @@ -1,403 +0,0 @@ -""" -Interactive config generation and editing. -""" - -from pathlib import Path -from typing import TYPE_CHECKING - -import yaml -from rich.console import Console - -from checkup.configuration.io import CONFIG_FILENAME -from checkup.configuration.schema import write_schema -from checkup.registry import get_registry - -if TYPE_CHECKING: - import questionary - - from checkup.configuration import CheckupConfig - from checkup.registry import PluginRegistry - -console = Console(markup=False, highlight=False) - - -def _get_questionary() -> "questionary": - """ - Lazy import questionary to avoid slow startup. - """ - - import questionary - - return questionary - - -def create_config(output_path: Path | None = None) -> None: - """ - Interactively create a new config file. - """ - - path = output_path or Path.cwd() / CONFIG_FILENAME - - if path.exists(): - overwrite = ( - _get_questionary() - .confirm(f"{path} exists. Overwrite?", default=False) - .ask() - ) - if overwrite is None or not overwrite: - if overwrite is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - - registry = get_registry() - config: dict = {} - - # Tags - console.print("\n[bold]Tags[/bold]", markup=True) - console.print( - "Tags identify your data product (e.g., product=my-product, team=analytics)" - ) - tags = _collect_tags({}) - if tags is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if tags: - config["tags"] = tags - - # Providers - console.print("\n[bold]Providers[/bold]", markup=True) - provider_names = _select_multiple(registry.list_provider_names(), [], "providers") - if provider_names is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if provider_names: - config["providers"] = [{"name": p} for p in provider_names] - - # Metrics - console.print("\n[bold]Metrics[/bold]", markup=True) - with console.status("Loading metrics..."): - available_metrics = registry.list_compatible_metric_names(provider_names or []) - metric_names = _select_multiple(available_metrics, [], "metrics") - if metric_names is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if metric_names: - config["metrics"] = [{"name": m} for m in metric_names] - - # Materializer - console.print("\n[bold]Materializer[/bold]", markup=True) - mat = _select_materializer(None, registry) - if mat is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if mat != "console": - config["materializer"] = {"type": mat} - - # Write - _write_config(path, config) - - -def edit_config(config_path: Path | None = None) -> None: - """ - Interactively edit an existing config file. - """ - - from checkup.configuration import load_config - - path = config_path or Path.cwd() / CONFIG_FILENAME - - if not path.exists(): - console.print(f"[red]Config file not found: {path}[/red]", markup=True) - console.print("Run [bold]checkup init[/bold] to create one.", markup=True) - return - - cfg = load_config(config_path=path) - registry = get_registry() - - console.print(f"[bold]Editing {path}[/bold]\n", markup=True) - _show_current_config(cfg) - - new_config: dict = {} - - # Build lookup for existing configs - existing_provider_configs = {p.name: p.config for p in cfg.providers} - existing_metric_configs = {m.name: m.config for m in cfg.metrics} - - # Tags - edit_tags = _get_questionary().confirm("Edit tags?", default=False).ask() - if edit_tags is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if edit_tags: - tags = _edit_tags(dict(cfg.tags)) - if tags is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if tags: - new_config["tags"] = tags - elif cfg.tags: - new_config["tags"] = dict(cfg.tags) - - # Providers - current_provider_names = [p.name for p in cfg.providers] - edit_providers = _get_questionary().confirm("Edit providers?", default=False).ask() - if edit_providers is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if edit_providers: - provider_names = _select_multiple( - registry.list_provider_names(), - current_provider_names, - "providers", - ) - if provider_names is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if provider_names: - # Preserve existing config for re-selected providers - new_config["providers"] = [ - {"name": p, **existing_provider_configs.get(p, {})} - for p in provider_names - ] - else: - provider_names = current_provider_names - if provider_names: - new_config["providers"] = [ - {"name": p.name, **p.config} for p in cfg.providers - ] - - # Metrics - current_metric_names = [m.name for m in cfg.metrics] - with console.status("Loading metrics..."): - available_metrics = registry.list_compatible_metric_names(provider_names) - edit_metrics = _get_questionary().confirm("Edit metrics?", default=False).ask() - if edit_metrics is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if edit_metrics: - metric_names = _select_multiple( - available_metrics, - current_metric_names, - "metrics", - ) - if metric_names is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if metric_names: - # Preserve existing config for re-selected metrics - new_config["metrics"] = [ - {"name": m, **existing_metric_configs.get(m, {})} for m in metric_names - ] - elif current_metric_names: - new_config["metrics"] = [ - {"name": m.name, **m.config} - for m in cfg.metrics - if m.name in available_metrics - ] - - # Materializer - edit_mat = _get_questionary().confirm("Edit materializer?", default=False).ask() - if edit_mat is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - if edit_mat: - mat = _select_materializer( - cfg.materializer.type if cfg.materializer else None, - registry, - ) - if mat is None: - console.print("\n[yellow]Cancelled.[/yellow]", markup=True) - return - new_config["materializer"] = {"type": mat} - elif cfg.materializer: - new_config["materializer"] = {"type": cfg.materializer.type} - - _write_config(path, new_config) - - -def _show_current_config(cfg: "CheckupConfig") -> None: - """ - Display current configuration. - """ - - console.print("[bold]Current configuration:[/bold]", markup=True) - console.print(f" Tags: {dict(cfg.tags) or '(none)'}") - console.print(f" Providers: {[p.name for p in cfg.providers] or '(none)'}") - console.print(f" Metrics: {[m.name for m in cfg.metrics] or '(none)'}") - mat = cfg.materializer.type if cfg.materializer else "(none)" - console.print(f" Materializer: {mat}") - console.print() - - -def _collect_tags(existing: dict) -> dict | None: - """ - Collect tags interactively. Returns None if cancelled. - """ - - tags = dict(existing) - - while True: - tag = ( - _get_questionary() - .text( - "Add tag (key=value, or empty to finish):", - ) - .ask() - ) - - if tag is None: - return None - if not tag: - break - if "=" in tag: - key, value = tag.split("=", 1) - tags[key.strip()] = value.strip() - - return tags - - -def _edit_tags(tags: dict) -> dict | None: - """ - Edit tags interactively. Returns None if cancelled. - """ - - console.print(f"Current tags: {tags or '(none)'}") - - while True: - action = ( - _get_questionary() - .select( - "Action:", - choices=["done", "add", "remove"], - ) - .ask() - ) - - if action is None: - return None - if action == "done": - break - elif action == "add": - tag = _get_questionary().text("Tag (key=value):").ask() - if tag is None: - return None - if tag and "=" in tag: - key, value = tag.split("=", 1) - tags[key.strip()] = value.strip() - elif action == "remove" and tags: - key = ( - _get_questionary() - .select( - "Key to remove:", - choices=list(tags.keys()), - ) - .ask() - ) - if key is None: - return None - if key: - tags.pop(key, None) - - return tags - - -def _select_multiple( - available: list[str], - selected: list[str], - item_type: str, -) -> list[str]: - """ - Select multiple items with fuzzy search. - """ - - if not available: - console.print( - f"[yellow]No {item_type} found. Install checkup plugins.[/yellow]", - markup=True, - ) - return [] - - # Show current state - selected_count = len([s for s in selected if s in available]) - console.print(f"Currently selected: {selected_count}/{len(available)} {item_type}") - - choices = [ - _get_questionary().Choice(name, checked=name in selected) - for name in sorted(available) - ] - - result = ( - _get_questionary() - .checkbox( - f"Select {item_type}:", - choices=choices, - use_search_filter=True, - use_jk_keys=False, - instruction="(↑↓ navigate, space toggle, type to filter, enter confirm)", - ) - .ask() - ) - - return result or [] - - -def _select_materializer(current: str | None, registry: "PluginRegistry") -> str | None: - """ - Select materializer interactively. - """ - - available = registry.list_materializer_names() - - if not available: - return "console" - - default = ( - current if current in available else (available[0] if available else "console") - ) - - return ( - _get_questionary() - .select( - "Materializer type:", - choices=available, - default=default, - use_search_filter=True, - use_jk_keys=False, - ) - .ask() - ) - - -def _write_config(path: Path, config: dict) -> None: - """ - Write config to file with empty lines between sections. - Also generates the JSON schema file. - """ - - console.print(f"\n[bold]Writing config to {path}[/bold]", markup=True) - - # Write sections separately with blank lines between them - lines = ["# yaml-language-server: $schema=checkup.schema.json"] - - for key in ["tags", "providers", "metrics", "materializer"]: - if key in config: - lines.append("") - lines.append( - yaml.dump( - {key: config[key]}, default_flow_style=False, sort_keys=False - ).rstrip() - ) - - with open(path, "w") as f: - f.write("\n".join(lines)) - f.write("\n") - - # Generate schema file - schema_path = path.parent / "checkup.schema.json" - write_schema(schema_path) - console.print(f"[green]Schema written to {schema_path}[/green]", markup=True) - - console.print("[green]Done![/green]", markup=True) - console.print( - "Run [bold]checkup run[/bold] to test your configuration.", markup=True - ) diff --git a/src/checkup/cli/config_wizard/__init__.py b/src/checkup/cli/config_wizard/__init__.py new file mode 100644 index 0000000..a1048eb --- /dev/null +++ b/src/checkup/cli/config_wizard/__init__.py @@ -0,0 +1,8 @@ +""" +Interactive config generation and editing. +""" + +from .create import create_config +from .edit import edit_config + +__all__ = ["create_config", "edit_config"] diff --git a/src/checkup/cli/config_wizard/_common.py b/src/checkup/cli/config_wizard/_common.py new file mode 100644 index 0000000..cb4ec37 --- /dev/null +++ b/src/checkup/cli/config_wizard/_common.py @@ -0,0 +1,133 @@ +""" +Shared utilities for the config wizard. +""" + +from pathlib import Path +from typing import TYPE_CHECKING + +import yaml +from rich.console import Console + +from checkup.configuration.io import SCHEMA_FILENAME +from checkup.configuration.models import CheckupConfig +from checkup.configuration.schema import write_schema + +if TYPE_CHECKING: + import questionary + + from checkup.registry import PluginRegistry + +console = Console(markup=False, highlight=False) + + +def get_questionary() -> "questionary": + """ + Lazy import questionary to avoid slow startup. + """ + + import questionary + + return questionary + + +def select_multiple( + available: list[str], + selected: list[str], + item_name: str, +) -> list[str]: + """ + Select multiple items with fuzzy search. + """ + + if not available: + console.print( + f"[yellow]No {item_name} found. Install checkup plugins.[/yellow]", + markup=True, + ) + return [] + + selected_count = len([s for s in selected if s in available]) + console.print(f"Currently selected: {selected_count}/{len(available)} {item_name}") + + choices = [ + get_questionary().Choice(name, checked=name in selected) + for name in sorted(available) + ] + + result = ( + get_questionary() + .checkbox( + f"Select {item_name}:", + choices=choices, + use_search_filter=True, + use_jk_keys=False, + instruction="(↑↓ navigate, space toggle, type to filter, enter confirm)", + ) + .ask() + ) + + return result or [] + + +def select_materializer(current: str | None, registry: "PluginRegistry") -> str | None: + """ + Select materializer interactively. + """ + + available = registry.list_materializer_names() + + if not available: + return "console" + + default = ( + current if current in available else (available[0] if available else "console") + ) + + return ( + get_questionary() + .select( + "Materializer type:", + choices=available, + default=default, + use_search_filter=True, + use_jk_keys=False, + ) + .ask() + ) + + +def write_config(path: Path, config: CheckupConfig) -> None: + """ + Write config to file with empty lines between sections. + Also generates the JSON schema file. + """ + + console.print(f"\n[bold]Writing config to {path}[/bold]", markup=True) + + data = config.model_dump(exclude_defaults=True) + + lines = [f"# yaml-language-server: $schema={SCHEMA_FILENAME}"] + + # Write sections separately with blank lines between them + for key in CheckupConfig.model_fields: + if key not in data or not data[key]: + continue + lines.append("") + lines.append( + yaml.dump( + {key: data[key]}, default_flow_style=False, sort_keys=False + ).rstrip() + ) + + with open(path, "w") as f: + f.write("\n".join(lines)) + f.write("\n") + + schema_path = path.parent / SCHEMA_FILENAME + write_schema(schema_path) + console.print(f"[green]Schema written to {schema_path}[/green]", markup=True) + + console.print("[green]Done![/green]", markup=True) + console.print( + "Run [bold]checkup run[/bold] to test your configuration.", markup=True + ) diff --git a/src/checkup/cli/config_wizard/create.py b/src/checkup/cli/config_wizard/create.py new file mode 100644 index 0000000..77171cb --- /dev/null +++ b/src/checkup/cli/config_wizard/create.py @@ -0,0 +1,134 @@ +""" +Interactive config creation. +""" + +from pathlib import Path +from typing import TYPE_CHECKING + +from checkup.configuration.io import CONFIG_FILENAME +from checkup.configuration.models import ( + CheckupConfig, + MaterializerConfig, + MetricConfig, + ProviderConfig, +) +from checkup.registry import get_registry + +from ._common import ( + console, + get_questionary, + select_materializer, + select_multiple, + write_config, +) + +if TYPE_CHECKING: + from checkup.registry import PluginRegistry + + +def create_config(output_path: Path | None = None) -> None: + """ + Interactively create a new config file. + """ + + path = output_path or Path.cwd() / CONFIG_FILENAME + + if not _confirm_overwrite(path): + return + + registry = get_registry() + config = _build_config(registry) + + if config is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + + write_config(path, config) + + +def _confirm_overwrite(path: Path) -> bool: + if not path.exists(): + return True + + overwrite = ( + get_questionary().confirm(f"{path} exists. Overwrite?", default=False).ask() + ) + if overwrite is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return False + return overwrite + + +def _build_config(registry: "PluginRegistry") -> CheckupConfig | None: + tags = _prompt_tags() + if tags is None: + return None + + provider_names = _prompt_providers(registry) + if provider_names is None: + return None + + metric_names = _prompt_metrics(registry, provider_names) + if metric_names is None: + return None + + mat = _prompt_materializer(registry) + if mat is None: + return None + + return CheckupConfig( + tags=tags, + providers=[ProviderConfig(name=p) for p in provider_names], + metrics=[MetricConfig(name=m) for m in metric_names], + materializer=MaterializerConfig(type=mat) if mat != "console" else None, + ) + + +def _prompt_tags() -> dict[str, str] | None: + console.print("\n[bold]Tags[/bold]", markup=True) + console.print( + "Tags identify your data product (e.g., product=my-product, team=analytics)" + ) + return _collect_tags() + + +def _prompt_providers(registry: "PluginRegistry") -> list[str] | None: + console.print("\n[bold]Providers[/bold]", markup=True) + return select_multiple(registry.list_provider_names(), [], "providers") + + +def _prompt_metrics( + registry: "PluginRegistry", provider_names: list[str] +) -> list[str] | None: + console.print("\n[bold]Metrics[/bold]", markup=True) + with console.status("Loading metrics..."): + available_metrics = registry.list_compatible_metric_names(provider_names) + return select_multiple(available_metrics, [], "metrics") + + +def _prompt_materializer(registry: "PluginRegistry") -> str | None: + console.print("\n[bold]Materializer[/bold]", markup=True) + return select_materializer(None, registry) + + +def _collect_tags() -> dict[str, str] | None: + tags: dict[str, str] = {} + + while True: + tag: str = ( + get_questionary() + .text( + "Add tag (key=value, or empty to finish):", + ) + .ask() + ) + + if tag is None: + return None + if not tag: + break + if "=" in tag: + key, value = tag.split("=", 1) + tags[key.strip()] = value.strip() + + return tags diff --git a/src/checkup/cli/config_wizard/edit.py b/src/checkup/cli/config_wizard/edit.py new file mode 100644 index 0000000..56b6ac5 --- /dev/null +++ b/src/checkup/cli/config_wizard/edit.py @@ -0,0 +1,200 @@ +""" +Interactive config editing. +""" + +from pathlib import Path +from typing import TYPE_CHECKING + +from checkup.configuration import load_config +from checkup.configuration.io import CONFIG_FILENAME +from checkup.configuration.models import ( + CheckupConfig, + MaterializerConfig, + MetricConfig, + ProviderConfig, +) +from checkup.registry import get_registry + +from ._common import ( + console, + get_questionary, + select_materializer, + select_multiple, + write_config, +) + +if TYPE_CHECKING: + from checkup.registry import PluginRegistry + + +def edit_config(config_path: Path | None = None) -> None: + """ + Interactively edit an existing config file. + """ + + path = config_path or Path.cwd() / CONFIG_FILENAME + + if not path.exists(): + console.print(f"[red]Config file not found: {path}[/red]", markup=True) + console.print("Run [bold]checkup init[/bold] to create one.", markup=True) + return + + config = load_config(config_path=path) + registry = get_registry() + + console.print(f"[bold]Editing {path}[/bold]\n", markup=True) + _show_current_config(config) + + config_new = _build_config(config, registry) + + if config_new is None: + console.print("\n[yellow]Cancelled.[/yellow]", markup=True) + return + + write_config(path, config_new) + + +def _build_config( + config: CheckupConfig, registry: "PluginRegistry" +) -> CheckupConfig | None: + provider_configs = {p.name: p.config for p in config.providers} + metric_configs = {m.name: m.config for m in config.metrics} + + tags = _prompt_edit_tags(config) + if tags is None: + return None + + provider_names = _prompt_edit_providers(config, registry) + if provider_names is None: + return None + + metric_names = _prompt_edit_metrics(config, registry, provider_names) + if metric_names is None: + return None + + mat = _prompt_edit_materializer(config, registry) + if mat is None: + return None + + return CheckupConfig( + tags=tags, + providers=[ + ProviderConfig(name=p, config=provider_configs.get(p, {})) + for p in provider_names + ], + metrics=[ + MetricConfig(name=m, config=metric_configs.get(m, {})) for m in metric_names + ], + materializer=MaterializerConfig(type=mat) if mat else None, + ) + + +def _prompt_edit_tags(config: "CheckupConfig") -> dict | None: + edit = get_questionary().confirm("Edit tags?", default=False).ask() + if edit is None: + return None + elif edit: + return _edit_tags(dict(config.tags)) + return dict(config.tags) + + +def _prompt_edit_providers( + config: "CheckupConfig", + registry: "PluginRegistry", +) -> list[str] | None: + current_names = [p.name for p in config.providers] + + edit = get_questionary().confirm("Edit providers?", default=False).ask() + if edit is None: + return None + elif edit: + return select_multiple( + registry.list_provider_names(), + current_names, + "providers", + ) + return current_names + + +def _prompt_edit_metrics( + config: "CheckupConfig", + registry: "PluginRegistry", + provider_names: list[str], +) -> list[str] | None: + current_names = [m.name for m in config.metrics] + + with console.status("Loading metrics..."): + available = registry.list_compatible_metric_names(provider_names) + + edit = get_questionary().confirm("Edit metrics?", default=False).ask() + if edit is None: + return None + elif edit: + return select_multiple(available, current_names, "metrics") + return [m.name for m in config.metrics if m.name in available] + + +def _prompt_edit_materializer( + config: "CheckupConfig", + registry: "PluginRegistry", +) -> str | None: + edit = get_questionary().confirm("Edit materializer?", default=False).ask() + if edit is None: + return None + elif edit: + return select_materializer( + config.materializer.type if config.materializer else None, + registry, + ) + return config.materializer.type if config.materializer else "" + + +def _edit_tags(tags: dict) -> dict | None: + console.print(f"Current tags: {tags or '(none)'}") + + while True: + action: str | None = ( + get_questionary() + .select( + "Action:", + choices=["done", "add", "remove"], + ) + .ask() + ) + + if action is None: + return None + if action == "done": + break + elif action == "add": + tag: str | None = get_questionary().text("Tag (key=value):").ask() + if tag is None: + return None + if tag and "=" in tag: + key, value = tag.split("=", 1) + tags[key.strip()] = value.strip() + elif action == "remove" and tags: + key = ( + get_questionary() + .select( + "Key to remove:", + choices=list(tags.keys()), + ) + .ask() + ) + if key is None: + return None + if key: + tags.pop(key, None) + + return tags + + +def _show_current_config(config: "CheckupConfig") -> None: + console.print("[bold]Current configuration:[/bold]", markup=True) + console.print(f" Tags: {dict(config.tags) or '(none)'}") + console.print(f" Providers: {[p.name for p in config.providers] or '(none)'}") + console.print(f" Metrics: {[m.name for m in config.metrics] or '(none)'}") + mat = config.materializer.type if config.materializer else "(none)" + console.print(f" Materializer: {mat}") + console.print() diff --git a/src/checkup/configuration/io.py b/src/checkup/configuration/io.py index 527d725..1ae4823 100644 --- a/src/checkup/configuration/io.py +++ b/src/checkup/configuration/io.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) CONFIG_FILENAME = "checkup.yaml" +SCHEMA_FILENAME = "checkup.schema.json" def load_yaml_file(path: Path) -> dict[str, Any]: From 32a44b184564a3b665d707708efd6c24e7a2054b Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Thu, 23 Apr 2026 10:25:06 +0200 Subject: [PATCH 20/21] parsecliitem tests --- src/checkup/cli/utils.py | 14 ++++++++++---- tests/test_cli_configuration.py | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/checkup/cli/utils.py b/src/checkup/cli/utils.py index e2e5da4..32ed4bd 100644 --- a/src/checkup/cli/utils.py +++ b/src/checkup/cli/utils.py @@ -2,8 +2,12 @@ CLI utility functions. """ +import logging + from checkup.configuration import CheckupConfig, MetricConfig, ProviderConfig +logger = logging.getLogger(__name__) + def apply_cli_overrides( cfg: CheckupConfig, @@ -59,11 +63,13 @@ def parse_cli_item(item: str) -> tuple[str, dict]: return item, {} name, config_str = item.split(":", 1) - config = {} + config: dict[str, str] = {} for pair in config_str.split(","): - if "=" in pair: - key, value = pair.split("=", 1) - config[key] = value + if "=" not in pair: + logger.warning("Ignoring malformed config pair %r in %r", pair, item) + continue + key, value = pair.split("=", 1) + config[key] = value return name, config diff --git a/tests/test_cli_configuration.py b/tests/test_cli_configuration.py index 7e87e87..fc10496 100644 --- a/tests/test_cli_configuration.py +++ b/tests/test_cli_configuration.py @@ -4,6 +4,7 @@ import yaml +from checkup.cli.utils import parse_cli_item from checkup.configuration.env import ( apply_naming_convention_overrides, substitute_env_vars, @@ -250,3 +251,36 @@ def test_hierarchical_loading_merges_parent_and_child(self, tmp_path): result = load_config(start_dir=child_dir) assert result.tags == {"team": "platform", "product": "my-product"} + + +class TestParseCliItem: + def test_name_only(self): + name, config = parse_cli_item("git") + + assert name == "git" + assert config == {} + + def test_empty_config(self): + name, config = parse_cli_item("git:") + + assert name == "git" + assert config == {} + + def test_name_with_config_pairs(self): + name, config = parse_cli_item("dbt:project_dir=./dbt,profiles_dir=~/.dbt") + + assert name == "dbt" + assert config == {"project_dir": "./dbt", "profiles_dir": "~/.dbt"} + + def test_value_containing_special_characters(self): + name, config = parse_cli_item("db:url=postgres://host:5432,user=name=admin") + + assert name == "db" + assert config == {"url": "postgres://host:5432", "user": "name=admin"} + + def test_malformed_pair_is_skipped(self, caplog): + name, config = parse_cli_item("dbt:project_dir=./dbt,malformed,other=value") + + assert name == "dbt" + assert config == {"project_dir": "./dbt", "other": "value"} + assert "malformed" in caplog.text From 3f3c545adb566b183ba72c5eba15a26bf0afdd6d Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Thu, 23 Apr 2026 10:35:26 +0200 Subject: [PATCH 21/21] updated env var parsing logging --- src/checkup/configuration/env.py | 30 ++++++++++++++++++++++-------- tests/test_cli_configuration.py | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/checkup/configuration/env.py b/src/checkup/configuration/env.py index 75d5e52..bb9d63a 100644 --- a/src/checkup/configuration/env.py +++ b/src/checkup/configuration/env.py @@ -71,16 +71,30 @@ def apply_naming_convention_overrides(config: dict[str, Any]) -> dict[str, Any]: continue parts = key[len(prefix) :].lower().split("__") - if len(parts) < 2: - continue - section = parts[0] - if section == "materializer" and len(parts) >= 3: - _apply_materializer_override(config, parts, value, key) - - elif section == "provider" and len(parts) >= 3: - _apply_provider_override(config, parts, value, key) + if section == "materializer": + if len(parts) >= 3: + _apply_materializer_override(config, parts, value, key) + else: + logger.warning( + "Ignoring malformed env var %s (expected CHECKUP__MATERIALIZER____)", + key, + ) + + elif section == "provider": + if len(parts) >= 3: + _apply_provider_override(config, parts, value, key) + else: + logger.warning( + "Ignoring malformed env var %s (expected CHECKUP__PROVIDER____)", + key, + ) + + else: + logger.warning( + "Ignoring unknown env var %s (unknown section %r)", key, section + ) return config diff --git a/tests/test_cli_configuration.py b/tests/test_cli_configuration.py index fc10496..717a995 100644 --- a/tests/test_cli_configuration.py +++ b/tests/test_cli_configuration.py @@ -205,6 +205,24 @@ def test_explicit_config_wins_over_naming_convention(self, monkeypatch): assert result["materializer"]["connection_url"] == "yaml-url" + def test_malformed_materializer_env_var_logs_warning(self, monkeypatch, caplog): + monkeypatch.setenv("CHECKUP__MATERIALIZER__SQLALCHEMY", "value") + config = {"materializer": {"type": "sqlalchemy"}} + + apply_naming_convention_overrides(config) + + assert "malformed" in caplog.text.lower() + assert "CHECKUP__MATERIALIZER__SQLALCHEMY" in caplog.text + + def test_malformed_provider_env_var_logs_warning(self, monkeypatch, caplog): + monkeypatch.setenv("CHECKUP__PROVIDER__GIT", "value") + config = {"providers": [{"name": "git"}]} + + apply_naming_convention_overrides(config) + + assert "malformed" in caplog.text.lower() + assert "CHECKUP__PROVIDER__GIT" in caplog.text + class TestLoadConfig: def test_loads_yaml_file(self, tmp_path):