From 41ccd266f9d058ac31e3a01f2f58b339dae7c1e1 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 21:29:23 -0700 Subject: [PATCH 01/25] feat: add control template support --- models/src/agent_control_models/__init__.py | 28 + models/src/agent_control_models/controls.py | 214 ++++++- models/src/agent_control_models/errors.py | 16 + models/src/agent_control_models/server.py | 59 +- models/tests/test_control_templates.py | 99 +++ sdks/python/src/agent_control/__init__.py | 44 +- sdks/python/src/agent_control/controls.py | 63 +- sdks/python/tests/test_controls_api.py | 109 ++++ .../overlays/method-names.overlay.yaml | 5 + .../src/generated/funcs/controls-list.ts | 2 + .../funcs/controls-render-template.ts | 171 +++++ .../models/boolean-template-parameter.ts | 103 +++ .../models/control-definition-input.ts | 50 +- .../models/control-definition-output.ts | 46 +- .../src/generated/models/control-summary.ts | 6 + .../models/create-control-request.ts | 43 +- .../models/enum-template-parameter.ts | 112 ++++ sdks/typescript/src/generated/models/index.ts | 16 + .../src/generated/models/json-value-input.ts | 40 ++ .../src/generated/models/json-value-input1.ts | 47 ++ .../src/generated/models/json-value-output.ts | 44 ++ .../generated/models/json-value-output1.ts | 48 ++ .../list-controls-api-v1-controls-get.ts | 7 + .../models/regex-template-parameter.ts | 110 ++++ .../models/render-control-template-request.ts | 64 ++ .../render-control-template-response.ts | 45 ++ .../models/set-control-data-request.ts | 51 +- sdks/typescript/src/generated/models/step.ts | 30 +- .../models/string-list-template-parameter.ts | 112 ++++ .../models/string-template-parameter.ts | 110 ++++ .../models/template-control-input.ts | 62 ++ .../models/template-definition-input.ts | 67 ++ .../models/template-definition-output.ts | 62 ++ .../models/template-parameter-definition.ts | 98 +++ .../src/generated/models/template-value.ts | 39 ++ .../models/validate-control-data-request.ts | 53 +- sdks/typescript/src/generated/sdk/controls.ts | 19 + .../generated/types/discriminated-union.ts | 101 +++ .../src/generated/types/smart-union.ts | 3 +- .../endpoints/controls.py | 130 +++- server/src/agent_control_server/main.py | 6 + .../services/control_templates.py | 603 ++++++++++++++++++ server/tests/test_control_templates.py | 169 +++++ 43 files changed, 3225 insertions(+), 81 deletions(-) create mode 100644 models/tests/test_control_templates.py create mode 100644 sdks/python/tests/test_controls_api.py create mode 100644 sdks/typescript/src/generated/funcs/controls-render-template.ts create mode 100644 sdks/typescript/src/generated/models/boolean-template-parameter.ts create mode 100644 sdks/typescript/src/generated/models/enum-template-parameter.ts create mode 100644 sdks/typescript/src/generated/models/json-value-input.ts create mode 100644 sdks/typescript/src/generated/models/json-value-input1.ts create mode 100644 sdks/typescript/src/generated/models/json-value-output.ts create mode 100644 sdks/typescript/src/generated/models/json-value-output1.ts create mode 100644 sdks/typescript/src/generated/models/regex-template-parameter.ts create mode 100644 sdks/typescript/src/generated/models/render-control-template-request.ts create mode 100644 sdks/typescript/src/generated/models/render-control-template-response.ts create mode 100644 sdks/typescript/src/generated/models/string-list-template-parameter.ts create mode 100644 sdks/typescript/src/generated/models/string-template-parameter.ts create mode 100644 sdks/typescript/src/generated/models/template-control-input.ts create mode 100644 sdks/typescript/src/generated/models/template-definition-input.ts create mode 100644 sdks/typescript/src/generated/models/template-definition-output.ts create mode 100644 sdks/typescript/src/generated/models/template-parameter-definition.ts create mode 100644 sdks/typescript/src/generated/models/template-value.ts create mode 100644 sdks/typescript/src/generated/types/discriminated-union.ts create mode 100644 server/src/agent_control_server/services/control_templates.py create mode 100644 server/tests/test_control_templates.py diff --git a/models/src/agent_control_models/__init__.py b/models/src/agent_control_models/__init__.py index 8ef92d97..29eab30a 100644 --- a/models/src/agent_control_models/__init__.py +++ b/models/src/agent_control_models/__init__.py @@ -18,15 +18,27 @@ StepSchema, ) from .controls import ( + BooleanTemplateParameter, ConditionNode, ControlAction, ControlDefinition, + ControlDefinitionRuntime, ControlMatch, ControlScope, ControlSelector, + EnumTemplateParameter, EvaluatorResult, EvaluatorSpec, + JsonValue, + RegexTemplateParameter, SteeringContext, + StringListTemplateParameter, + StringTemplateParameter, + TemplateControlInput, + TemplateDefinition, + TemplateParameterBase, + TemplateParameterDefinition, + TemplateValue, ) from .errors import ( ERROR_TITLES, @@ -74,6 +86,8 @@ PaginationInfo, PatchControlRequest, PatchControlResponse, + RenderControlTemplateRequest, + RenderControlTemplateResponse, StepKey, ValidateControlDataRequest, ValidateControlDataResponse, @@ -104,9 +118,21 @@ "ControlMatch", "ControlScope", "ControlSelector", + "ControlDefinitionRuntime", "EvaluatorSpec", "EvaluatorResult", "SteeringContext", + "JsonValue", + "TemplateValue", + "TemplateParameterBase", + "StringTemplateParameter", + "StringListTemplateParameter", + "EnumTemplateParameter", + "BooleanTemplateParameter", + "RegexTemplateParameter", + "TemplateParameterDefinition", + "TemplateDefinition", + "TemplateControlInput", # Error models "ProblemDetail", "ErrorCode", @@ -132,6 +158,8 @@ "PaginationInfo", "PatchControlRequest", "PatchControlResponse", + "RenderControlTemplateRequest", + "RenderControlTemplateResponse", "StepKey", "ValidateControlDataRequest", "ValidateControlDataResponse", diff --git a/models/src/agent_control_models/controls.py b/models/src/agent_control_models/controls.py index a043622b..f56ad6c7 100644 --- a/models/src/agent_control_models/controls.py +++ b/models/src/agent_control_models/controls.py @@ -2,14 +2,16 @@ from __future__ import annotations +import re from collections.abc import Iterator from dataclasses import dataclass -from typing import Any, Literal, Self +from typing import Annotated, Any, Literal, Self, cast from uuid import uuid4 import re2 from pydantic import ConfigDict, Field, ValidationInfo, field_validator, model_validator +from .agent import JSONValue from .base import BaseModel @@ -262,6 +264,171 @@ class SteeringContext(BaseModel): } +type TemplateValue = str | bool | list[str] +type JsonValue = JSONValue + +_TEMPLATE_PARAMETER_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") + + +def _validate_re2_value(value: str, *, field_name: str) -> str: + """Validate that a string is a valid RE2 expression.""" + try: + re2.compile(value) + except re2.error as exc: + raise ValueError(f"Invalid {field_name}: {exc}") from exc + return value + + +class TemplateParameterBase(BaseModel): + """Base definition for a template parameter.""" + + label: str = Field(..., min_length=1, description="Human-readable parameter label") + description: str | None = Field( + None, + description="Optional description of what the parameter controls", + ) + required: bool = Field( + True, + description="Whether the caller must provide a value when no default exists", + ) + ui_hint: str | None = Field( + None, + description="Optional UI hint for rendering the parameter input", + ) + + +class StringTemplateParameter(TemplateParameterBase): + """String-valued template parameter.""" + + type: Literal["string"] = "string" + default: str | None = Field(None, description="Optional default value") + placeholder: str | None = Field(None, description="Optional placeholder text") + + +class StringListTemplateParameter(TemplateParameterBase): + """List-of-strings template parameter.""" + + type: Literal["string_list"] = "string_list" + default: list[str] | None = Field(None, description="Optional default value") + placeholder: list[str] | None = Field( + None, + description="Optional placeholder/example list", + ) + + @field_validator("default", "placeholder") + @classmethod + def validate_string_lists( + cls, value: list[str] | None + ) -> list[str] | None: + if value is None: + return value + if any((not isinstance(item, str)) for item in value): + raise ValueError("Values must be strings") + return value + + +class EnumTemplateParameter(TemplateParameterBase): + """String enum template parameter.""" + + type: Literal["enum"] = "enum" + allowed_values: list[str] = Field( + ..., + min_length=1, + description="Allowed string values for the parameter", + ) + default: str | None = Field(None, description="Optional default value") + + @field_validator("allowed_values") + @classmethod + def validate_allowed_values(cls, value: list[str]) -> list[str]: + if not value: + raise ValueError("allowed_values must not be empty") + if any(not item for item in value): + raise ValueError("allowed_values must contain non-empty strings") + if len(set(value)) != len(value): + raise ValueError("allowed_values must not contain duplicates") + return value + + @model_validator(mode="after") + def validate_default(self) -> Self: + if self.default is not None and self.default not in self.allowed_values: + raise ValueError("Default must be one of allowed_values") + return self + + +class BooleanTemplateParameter(TemplateParameterBase): + """Boolean template parameter.""" + + type: Literal["boolean"] = "boolean" + default: bool | None = Field(None, description="Optional default value") + + +class RegexTemplateParameter(TemplateParameterBase): + """RE2 regex template parameter.""" + + type: Literal["regex_re2"] = "regex_re2" + default: str | None = Field(None, description="Optional default regex pattern") + placeholder: str | None = Field(None, description="Optional placeholder regex") + + @field_validator("default", "placeholder") + @classmethod + def validate_regex_values(cls, value: str | None) -> str | None: + if value is None: + return value + return _validate_re2_value(value, field_name="regex pattern") + + +type TemplateParameterDefinition = Annotated[ + StringTemplateParameter + | StringListTemplateParameter + | EnumTemplateParameter + | BooleanTemplateParameter + | RegexTemplateParameter, + Field(discriminator="type"), +] + + +class TemplateDefinition(BaseModel): + """Reusable template with typed parameters and a JSON definition template.""" + + description: str | None = Field( + None, + description="Metadata describing the template itself", + ) + parameters: dict[str, TemplateParameterDefinition] = Field( + default_factory=dict, + description="Typed parameter definitions keyed by parameter name", + ) + definition_template: JsonValue = Field( + ..., + description="Template payload containing $param binding objects", + ) + + @field_validator("parameters") + @classmethod + def validate_parameter_names( + cls, value: dict[str, TemplateParameterDefinition] + ) -> dict[str, TemplateParameterDefinition]: + for name in value: + if not _TEMPLATE_PARAMETER_NAME_RE.fullmatch(name): + raise ValueError( + "Parameter names must match [a-zA-Z_][a-zA-Z0-9_]*" + ) + return value + + +class TemplateControlInput(BaseModel): + """Template-backed input payload for control create/update requests.""" + + model_config = ConfigDict(extra="forbid") + + template: TemplateDefinition = Field(..., description="Template definition to render") + template_values: dict[str, TemplateValue] = Field( + default_factory=dict, + description="Template parameter values keyed by parameter name", + ) + + class ControlAction(BaseModel): """What to do when control matches.""" @@ -474,6 +641,14 @@ class ControlDefinition(BaseModel): # Metadata tags: list[str] = Field(default_factory=list, description="Tags for categorization") + template: TemplateDefinition | None = Field( + None, + description="Template metadata for template-backed controls", + ) + template_values: dict[str, TemplateValue] | None = Field( + None, + description="Resolved parameter values for template-backed controls", + ) @classmethod def canonicalize_payload(cls, data: Any) -> Any: @@ -527,6 +702,12 @@ def validate_condition_constraints(self) -> Self: raise ValueError( "Composite steer controls require action.steering_context" ) + has_template = self.template is not None + has_template_values = self.template_values is not None + if has_template != has_template_values: + raise ValueError( + "template and template_values must both be present or both absent" + ) return self def iter_condition_leaves(self) -> Iterator[ConditionNode]: @@ -603,6 +784,37 @@ def observability_identity(self) -> ControlObservabilityIdentity: } +class ControlDefinitionRuntime(BaseModel): + """Slim runtime control model that ignores template authoring metadata.""" + + model_config = ConfigDict(extra="ignore") + + description: str | None = Field(None, description="Detailed description of the control") + enabled: bool = Field(True, description="Whether this control is active") + execution: Literal["server", "sdk"] = Field( + ..., description="Where this control executes" + ) + scope: ControlScope = Field( + default_factory=ControlScope, + description="Which steps and stages this control applies to", + ) + condition: ConditionNode = Field( + ..., + description=( + "Recursive boolean condition tree. Leaf nodes contain selector + evaluator; " + "composite nodes contain and/or/not." + ), + ) + action: ControlAction = Field(..., description="What action to take when control matches") + tags: list[str] = Field(default_factory=list, description="Tags for categorization") + + def to_control_definition(self) -> ControlDefinition: + """Promote a runtime control to the full public model when needed.""" + return ControlDefinition.model_validate( + cast(dict[str, JSONValue], self.model_dump(mode="python")) + ) + + class EvaluatorResult(BaseModel): """Result from a control evaluator. diff --git a/models/src/agent_control_models/errors.py b/models/src/agent_control_models/errors.py index 8d2803b0..8f37b08b 100644 --- a/models/src/agent_control_models/errors.py +++ b/models/src/agent_control_models/errors.py @@ -77,6 +77,8 @@ class ErrorCode(StrEnum): INVALID_SCHEMA = "INVALID_SCHEMA" CORRUPTED_DATA = "CORRUPTED_DATA" POLICY_CONTROL_INCOMPATIBLE = "POLICY_CONTROL_INCOMPATIBLE" + TEMPLATE_PARAMETER_INVALID = "TEMPLATE_PARAMETER_INVALID" + TEMPLATE_RENDER_ERROR = "TEMPLATE_RENDER_ERROR" # Server Errors (5xx pattern) DATABASE_ERROR = "DATABASE_ERROR" @@ -135,6 +137,18 @@ class ValidationErrorItem(BaseModel): default=None, description="The invalid value that was provided (omitted for sensitive data)", ) + parameter: str | None = Field( + default=None, + description="Template parameter key when the error maps to a template input", + ) + parameter_label: str | None = Field( + default=None, + description="Human-readable template parameter label for template-aware errors", + ) + rendered_field: str | None = Field( + default=None, + description="Rendered control field path that produced the validation error", + ) class ErrorMetadata(BaseModel): @@ -364,6 +378,8 @@ def make_error_type(error_code: ErrorCode) -> str: ErrorCode.INVALID_SCHEMA: "Invalid Schema", ErrorCode.CORRUPTED_DATA: "Corrupted Data", ErrorCode.POLICY_CONTROL_INCOMPATIBLE: "Policy Control Incompatible", + ErrorCode.TEMPLATE_PARAMETER_INVALID: "Template Parameter Invalid", + ErrorCode.TEMPLATE_RENDER_ERROR: "Template Render Error", # Server errors ErrorCode.DATABASE_ERROR: "Database Error", ErrorCode.INTERNAL_ERROR: "Internal Server Error", diff --git a/models/src/agent_control_models/server.py b/models/src/agent_control_models/server.py index b047a3a4..c7386cad 100644 --- a/models/src/agent_control_models/server.py +++ b/models/src/agent_control_models/server.py @@ -1,11 +1,11 @@ from enum import StrEnum from typing import Annotated, Any -from pydantic import BeforeValidator, Field, StringConstraints +from pydantic import BeforeValidator, Field, StringConstraints, TypeAdapter from .agent import Agent, StepSchema from .base import BaseModel -from .controls import ControlDefinition +from .controls import ControlDefinition, TemplateControlInput, TemplateDefinition, TemplateValue from .policy import Control @@ -14,6 +14,26 @@ def _strip_slug_name(v: str) -> str: return v.strip() if isinstance(v, str) else v +_CONTROL_DEFINITION_ADAPTER = TypeAdapter(ControlDefinition) +_TEMPLATE_CONTROL_INPUT_ADAPTER = TypeAdapter(TemplateControlInput) + + +def _parse_control_input(v: Any) -> Any: + """Discriminate raw control inputs from template-backed inputs. + + A non-null ``template`` key means template-backed input and must be parsed + strictly as ``TemplateControlInput`` so mixed payloads are rejected. + """ + if isinstance(v, (ControlDefinition, TemplateControlInput)): + return v + if not isinstance(v, dict): + return v + + if v.get("template") is not None: + return _TEMPLATE_CONTROL_INPUT_ADAPTER.validate_python(v) + return _CONTROL_DEFINITION_ADAPTER.validate_python(v) + + # Canonicalization at the API boundary: all SlugName fields are trimmed before # validation. Server and SDKs use these request models; no client need pre-trim. SlugName = Annotated[ @@ -26,6 +46,11 @@ def _strip_slug_name(v: str) -> str: ), ] +ControlInput = Annotated[ + ControlDefinition | TemplateControlInput, + BeforeValidator(_parse_control_input), +] + class EvaluatorSchema(BaseModel): """Schema for a custom evaluator registered with an agent. @@ -119,7 +144,7 @@ class CreateControlRequest(BaseModel): ..., description="Unique control name (letters, numbers, hyphens, underscores)", ) - data: ControlDefinition = Field( + data: ControlInput = Field( ..., description="Control definition to validate and store during creation", ) @@ -295,7 +320,7 @@ class GetControlDataResponse(BaseModel): class SetControlDataRequest(BaseModel): """Request to update control configuration data.""" - data: ControlDefinition = Field( + data: ControlInput = Field( ..., description="Control configuration data (replaces existing)", ) @@ -304,7 +329,7 @@ class SetControlDataRequest(BaseModel): class ValidateControlDataRequest(BaseModel): """Request to validate control configuration data without saving.""" - data: ControlDefinition = Field( + data: ControlInput = Field( ..., description="Control configuration data to validate", ) @@ -318,6 +343,25 @@ class ValidateControlDataResponse(BaseModel): success: bool = Field(description="Whether the control data is valid") +class RenderControlTemplateRequest(BaseModel): + """Request to render a template-backed control without persisting it.""" + + template: TemplateDefinition = Field(..., description="Template definition to render") + template_values: dict[str, TemplateValue] = Field( + default_factory=dict, + description="Template parameter values used during rendering", + ) + + +class RenderControlTemplateResponse(BaseModel): + """Rendered template preview response.""" + + control: ControlDefinition = Field( + ..., + description="Rendered control definition including template metadata", + ) + + class StepKey(BaseModel): """Identifies a registered step schema by type and name.""" @@ -402,6 +446,10 @@ class ControlSummary(BaseModel): step_types: list[str] | None = Field(None, description="Step types in scope") stages: list[str] | None = Field(None, description="Evaluation stages in scope") tags: list[str] = Field(default_factory=list, description="Control tags") + template_backed: bool = Field( + False, + description="Whether the control was created from a template", + ) used_by_agent: AgentRef | None = Field(None, description="Agent using this control") # TODO: Follow-up with full `used_by_agents` list for richer attribution. used_by_agents_count: int = Field( @@ -452,4 +500,3 @@ class PatchControlResponse(BaseModel): enabled: bool | None = Field( None, description="Current enabled status (if control has data configured)" ) - diff --git a/models/tests/test_control_templates.py b/models/tests/test_control_templates.py new file mode 100644 index 00000000..24b85333 --- /dev/null +++ b/models/tests/test_control_templates.py @@ -0,0 +1,99 @@ +"""Tests for template-backed control model contracts.""" + +from __future__ import annotations + +import pytest +from agent_control_models import ControlDefinition, TemplateControlInput, TemplateDefinition +from agent_control_models.server import CreateControlRequest +from pydantic import ValidationError + + +VALID_TEMPLATE = { + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + } + }, + "definition_template": { + "description": "Template-backed control", + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, +} + + +def test_control_definition_requires_template_fields_together() -> None: + with pytest.raises( + ValidationError, + match="template and template_values must both be present or both absent", + ): + ControlDefinition.model_validate( + { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "ok"}, + }, + }, + "action": {"decision": "deny"}, + "template": VALID_TEMPLATE, + } + ) + + +def test_template_definition_rejects_invalid_parameter_name() -> None: + with pytest.raises( + ValidationError, + match=r"Parameter names must match \[a-zA-Z_\]\[a-zA-Z0-9_\]\*", + ): + TemplateDefinition.model_validate( + { + "parameters": { + "bad.name": { + "type": "string", + "label": "Bad Name", + } + }, + "definition_template": {}, + } + ) + + +def test_create_control_request_parses_template_payload_as_template_input() -> None: + request = CreateControlRequest.model_validate( + { + "name": "template-control", + "data": { + "template": VALID_TEMPLATE, + "template_values": {"pattern": "hello"}, + }, + } + ) + + assert isinstance(request.data, TemplateControlInput) + + +def test_create_control_request_rejects_mixed_raw_and_template_payload() -> None: + with pytest.raises(ValidationError): + CreateControlRequest.model_validate( + { + "name": "template-control", + "data": { + "template": VALID_TEMPLATE, + "template_values": {"pattern": "hello"}, + "execution": "server", + }, + } + ) diff --git a/sdks/python/src/agent_control/__init__.py b/sdks/python/src/agent_control/__init__.py index 24353411..22c475e8 100644 --- a/sdks/python/src/agent_control/__init__.py +++ b/sdks/python/src/agent_control/__init__.py @@ -67,6 +67,9 @@ async def handle_input(user_message: str) -> str: EvaluatorSpec, Step, StepSchema, + TemplateControlInput, + TemplateDefinition, + TemplateValue, ) from . import agents, controls, evaluation, evaluators, policies @@ -871,6 +874,7 @@ async def list_controls( limit: int = 20, name: str | None = None, enabled: bool | None = None, + template_backed: bool | None = None, step_type: str | None = None, stage: Literal["pre", "post"] | None = None, execution: Literal["server", "sdk"] | None = None, @@ -886,6 +890,7 @@ async def list_controls( limit: Number of results per page (default 20, max 100) name: Optional filter by name (partial, case-insensitive) enabled: Optional filter by enabled status + template_backed: Optional filter by whether the control is template-backed step_type: Optional filter by step type (built-ins: 'tool', 'llm') stage: Optional filter by stage ('pre' or 'post') execution: Optional filter by execution ('server' or 'sdk') @@ -925,6 +930,7 @@ async def main(): limit=limit, name=name, enabled=enabled, + template_backed=template_backed, step_type=step_type, stage=stage, execution=execution, @@ -934,7 +940,7 @@ async def main(): async def create_control( name: str, - data: dict[str, Any] | ControlDefinition, + data: dict[str, Any] | ControlDefinition | TemplateControlInput, server_url: str | None = None, api_key: str | None = None, ) -> dict[str, Any]: @@ -943,7 +949,7 @@ async def create_control( Args: name: Unique name for the control - data: Control definition with a condition tree, action, scope, etc. + data: Raw control definition or template-backed control input server_url: Optional server URL (defaults to AGENT_CONTROL_URL env var) api_key: Optional API key for authentication (defaults to AGENT_CONTROL_API_KEY env var) @@ -988,6 +994,35 @@ async def main(): return await controls.create_control(client, name, data=data) +async def validate_control_data( + data: dict[str, Any] | ControlDefinition | TemplateControlInput, + server_url: str | None = None, + api_key: str | None = None, +) -> dict[str, Any]: + """Validate raw or template-backed control data without saving it.""" + _final_server_url = server_url or os.getenv('AGENT_CONTROL_URL') or 'http://localhost:8000' + + async with AgentControlClient(base_url=_final_server_url, api_key=api_key) as client: + return await controls.validate_control_data(client, data=data) + + +async def render_control_template( + template: dict[str, Any] | TemplateDefinition, + template_values: dict[str, TemplateValue], + server_url: str | None = None, + api_key: str | None = None, +) -> dict[str, Any]: + """Render a template-backed control preview without persisting it.""" + _final_server_url = server_url or os.getenv('AGENT_CONTROL_URL') or 'http://localhost:8000' + + async with AgentControlClient(base_url=_final_server_url, api_key=api_key) as client: + return await controls.render_control_template( + client, + template=template, + template_values=template_values, + ) + + async def get_control( control_id: int, server_url: str | None = None, @@ -1279,6 +1314,8 @@ async def main(): "get_control", "delete_control", "update_control", + "validate_control_data", + "render_control_template", # Decorator "control", "ControlViolationError", @@ -1324,10 +1361,13 @@ async def main(): "EvaluationRequest", "EvaluationResult", "ControlDefinition", + "TemplateControlInput", + "TemplateDefinition", "ControlSelector", "ControlScope", "ControlAction", "ControlMatch", "EvaluatorSpec", "EvaluatorResult", + "TemplateValue", ] diff --git a/sdks/python/src/agent_control/controls.py b/sdks/python/src/agent_control/controls.py index 767f2c86..bff95b0a 100644 --- a/sdks/python/src/agent_control/controls.py +++ b/sdks/python/src/agent_control/controls.py @@ -2,7 +2,12 @@ from typing import Any, Literal, cast -from agent_control_models import ControlDefinition +from agent_control_models import ( + ControlDefinition, + TemplateControlInput, + TemplateDefinition, + TemplateValue, +) from .client import AgentControlClient @@ -13,6 +18,7 @@ async def list_controls( limit: int = 20, name: str | None = None, enabled: bool | None = None, + template_backed: bool | None = None, step_type: str | None = None, stage: Literal["pre", "post"] | None = None, execution: Literal["server", "sdk"] | None = None, @@ -29,6 +35,7 @@ async def list_controls( limit: Maximum number of controls to return (default 20, max 100) name: Optional filter by name (partial, case-insensitive match) enabled: Optional filter by enabled status + template_backed: Optional filter by whether the control is template-backed step_type: Optional filter by step type (built-ins: 'tool', 'llm') stage: Optional filter by stage ('pre' or 'post') execution: Optional filter by execution ('server' or 'sdk') @@ -68,6 +75,8 @@ async def list_controls( params["name"] = name if enabled is not None: params["enabled"] = enabled + if template_backed is not None: + params["template_backed"] = template_backed if step_type is not None: params["step_type"] = step_type if stage is not None: @@ -118,7 +127,7 @@ async def get_control( async def create_control( client: AgentControlClient, name: str, - data: dict[str, Any] | ControlDefinition, + data: dict[str, Any] | ControlDefinition | TemplateControlInput, ) -> dict[str, Any]: """ Create a new control with a unique name and configuration. @@ -129,7 +138,7 @@ async def create_control( Args: client: AgentControlClient instance name: Unique name for the control - data: Control definition (condition tree, action, scope, etc.) + data: Raw control definition or template-backed control input Returns: Dictionary containing: @@ -163,7 +172,7 @@ async def create_control( print(f"Created and configured control: {result['control_id']}") """ payload: dict[str, Any] = {"name": name} - if isinstance(data, ControlDefinition): + if isinstance(data, (ControlDefinition, TemplateControlInput)): payload["data"] = data.model_dump(mode="json", exclude_none=True) else: payload["data"] = cast(dict[str, Any], data) @@ -182,7 +191,7 @@ async def create_control( async def set_control_data( client: AgentControlClient, control_id: int, - data: dict[str, Any] | ControlDefinition + data: dict[str, Any] | ControlDefinition | TemplateControlInput ) -> dict[str, Any]: """ Set the configuration data for a control. @@ -192,7 +201,7 @@ async def set_control_data( Args: client: AgentControlClient instance control_id: ID of the control - data: Control definition dictionary or Pydantic model + data: Raw control definition or template-backed control input Returns: Dictionary containing success flag @@ -201,7 +210,7 @@ async def set_control_data( httpx.HTTPError: If request fails HTTPException 422: If data doesn't match schema """ - if isinstance(data, ControlDefinition): + if isinstance(data, (ControlDefinition, TemplateControlInput)): # Convert model to dict, excluding None to keep payload clean payload: dict[str, Any] = data.model_dump(mode="json", exclude_none=True) else: @@ -216,6 +225,46 @@ async def set_control_data( return cast(dict[str, Any], response.json()) +async def validate_control_data( + client: AgentControlClient, + data: dict[str, Any] | ControlDefinition | TemplateControlInput, +) -> dict[str, Any]: + """Validate raw or template-backed control data without saving it.""" + if isinstance(data, (ControlDefinition, TemplateControlInput)): + payload: dict[str, Any] = data.model_dump(mode="json", exclude_none=True) + else: + payload = cast(dict[str, Any], data) + + response = await client.http_client.post( + "/api/v1/controls/validate", + json={"data": payload}, + ) + response.raise_for_status() + return cast(dict[str, Any], response.json()) + + +async def render_control_template( + client: AgentControlClient, + template: dict[str, Any] | TemplateDefinition, + template_values: dict[str, TemplateValue], +) -> dict[str, Any]: + """Render a control template preview without persisting it.""" + if isinstance(template, TemplateDefinition): + serialized_template = template.model_dump(mode="json", exclude_none=True) + else: + serialized_template = cast(dict[str, Any], template) + + response = await client.http_client.post( + "/api/v1/control-templates/render", + json={ + "template": serialized_template, + "template_values": template_values, + }, + ) + response.raise_for_status() + return cast(dict[str, Any], response.json()) + + async def add_rule_to_control( client: AgentControlClient, control_id: int, diff --git a/sdks/python/tests/test_controls_api.py b/sdks/python/tests/test_controls_api.py new file mode 100644 index 00000000..fbf15811 --- /dev/null +++ b/sdks/python/tests/test_controls_api.py @@ -0,0 +1,109 @@ +"""Unit tests for agent_control.controls API wrappers.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +import agent_control +from agent_control_models import TemplateControlInput + + +@pytest.mark.asyncio +async def test_list_controls_passes_template_backed_filter() -> None: + response = Mock() + response.raise_for_status = Mock() + response.json = Mock(return_value={"controls": [], "pagination": {}}) + client = SimpleNamespace(http_client=SimpleNamespace(get=AsyncMock(return_value=response))) + + await agent_control.controls.list_controls(client, template_backed=True) + + client.http_client.get.assert_awaited_once_with( + "/api/v1/controls", + params={"limit": 20, "template_backed": True}, + ) + + +@pytest.mark.asyncio +async def test_create_control_accepts_template_control_input() -> None: + response = Mock() + response.raise_for_status = Mock() + response.json = Mock(return_value={"control_id": 123}) + client = SimpleNamespace(http_client=SimpleNamespace(put=AsyncMock(return_value=response))) + template_input = TemplateControlInput.model_validate( + { + "template": { + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + } + }, + "definition_template": { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {"pattern": "hello"}, + } + ) + + await agent_control.controls.create_control(client, "templated", template_input) + + client.http_client.put.assert_awaited_once() + _, kwargs = client.http_client.put.await_args + assert kwargs["json"]["data"]["template_values"]["pattern"] == "hello" + + +@pytest.mark.asyncio +async def test_render_control_template_calls_preview_endpoint() -> None: + response = Mock() + response.raise_for_status = Mock() + response.json = Mock(return_value={"control": {"execution": "server"}}) + client = SimpleNamespace(http_client=SimpleNamespace(post=AsyncMock(return_value=response))) + + await agent_control.controls.render_control_template( + client, + template={ + "parameters": {}, + "definition_template": { + "execution": "server", + "scope": {}, + "condition": { + "selector": {"path": "input"}, + "evaluator": {"name": "regex", "config": {"pattern": "x"}}, + }, + "action": {"decision": "deny"}, + }, + }, + template_values={}, + ) + + client.http_client.post.assert_awaited_once_with( + "/api/v1/control-templates/render", + json={ + "template": { + "parameters": {}, + "definition_template": { + "execution": "server", + "scope": {}, + "condition": { + "selector": {"path": "input"}, + "evaluator": {"name": "regex", "config": {"pattern": "x"}}, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {}, + }, + ) diff --git a/sdks/typescript/overlays/method-names.overlay.yaml b/sdks/typescript/overlays/method-names.overlay.yaml index 1c27b227..ac9daebe 100644 --- a/sdks/typescript/overlays/method-names.overlay.yaml +++ b/sdks/typescript/overlays/method-names.overlay.yaml @@ -100,6 +100,11 @@ actions: x-speakeasy-group: agents x-speakeasy-name-override: updatePolicy + - target: $["paths"]["/api/v1/control-templates/render"]["post"] + update: + x-speakeasy-group: controls + x-speakeasy-name-override: renderTemplate + - target: $["paths"]["/api/v1/controls"]["get"] update: x-speakeasy-group: controls diff --git a/sdks/typescript/src/generated/funcs/controls-list.ts b/sdks/typescript/src/generated/funcs/controls-list.ts index 2689f384..2fd69073 100644 --- a/sdks/typescript/src/generated/funcs/controls-list.ts +++ b/sdks/typescript/src/generated/funcs/controls-list.ts @@ -40,6 +40,7 @@ import { Result } from "../types/fp.js"; * limit: Maximum number of controls to return (default 20, max 100) * name: Optional filter by name (partial, case-insensitive match) * enabled: Optional filter by enabled status + * template_backed: Optional filter by whether the control is template-backed * step_type: Optional filter by step type (built-ins: 'tool', 'llm') * stage: Optional filter by stage ('pre' or 'post') * execution: Optional filter by execution ('server' or 'sdk') @@ -126,6 +127,7 @@ async function $do( "stage": payload?.stage, "step_type": payload?.step_type, "tag": payload?.tag, + "template_backed": payload?.template_backed, }); const headers = new Headers(compactMap({ diff --git a/sdks/typescript/src/generated/funcs/controls-render-template.ts b/sdks/typescript/src/generated/funcs/controls-render-template.ts new file mode 100644 index 00000000..a8998d0e --- /dev/null +++ b/sdks/typescript/src/generated/funcs/controls-render-template.ts @@ -0,0 +1,171 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { AgentControlSDKCore } from "../core.js"; +import { encodeJSON } from "../lib/encodings.js"; +import * as M from "../lib/matchers.js"; +import { compactMap } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { RequestOptions } from "../lib/sdks.js"; +import { extractSecurity, resolveGlobalSecurity } from "../lib/security.js"; +import { pathToFunc } from "../lib/url.js"; +import { AgentControlSDKError } from "../models/errors/agent-control-sdk-error.js"; +import { + ConnectionError, + InvalidRequestError, + RequestAbortedError, + RequestTimeoutError, + UnexpectedClientError, +} from "../models/errors/http-client-errors.js"; +import * as errors from "../models/errors/index.js"; +import { ResponseValidationError } from "../models/errors/response-validation-error.js"; +import { SDKValidationError } from "../models/errors/sdk-validation-error.js"; +import * as models from "../models/index.js"; +import { APICall, APIPromise } from "../types/async.js"; +import { Result } from "../types/fp.js"; + +/** + * Render a control template preview + * + * @remarks + * Render a template-backed control without persisting it. + */ +export function controlsRenderTemplate( + client: AgentControlSDKCore, + request: models.RenderControlTemplateRequest, + options?: RequestOptions, +): APIPromise< + Result< + models.RenderControlTemplateResponse, + | errors.HTTPValidationError + | AgentControlSDKError + | ResponseValidationError + | ConnectionError + | RequestAbortedError + | RequestTimeoutError + | InvalidRequestError + | UnexpectedClientError + | SDKValidationError + > +> { + return new APIPromise($do( + client, + request, + options, + )); +} + +async function $do( + client: AgentControlSDKCore, + request: models.RenderControlTemplateRequest, + options?: RequestOptions, +): Promise< + [ + Result< + models.RenderControlTemplateResponse, + | errors.HTTPValidationError + | AgentControlSDKError + | ResponseValidationError + | ConnectionError + | RequestAbortedError + | RequestTimeoutError + | InvalidRequestError + | UnexpectedClientError + | SDKValidationError + >, + APICall, + ] +> { + const parsed = safeParse( + request, + (value) => + z.parse(models.RenderControlTemplateRequest$outboundSchema, value), + "Input validation failed", + ); + if (!parsed.ok) { + return [parsed, { status: "invalid" }]; + } + const payload = parsed.value; + const body = encodeJSON("body", payload, { explode: true }); + + const path = pathToFunc("/api/v1/control-templates/render")(); + + const headers = new Headers(compactMap({ + "Content-Type": "application/json", + Accept: "application/json", + })); + + const secConfig = await extractSecurity(client._options.apiKeyHeader); + const securityInput = secConfig == null ? {} : { apiKeyHeader: secConfig }; + const requestSecurity = resolveGlobalSecurity(securityInput); + + const context = { + options: client._options, + baseURL: options?.serverURL ?? client._baseURL ?? "", + operationID: "render_control_template_api_v1_control_templates_render_post", + oAuth2Scopes: null, + + resolvedSecurity: requestSecurity, + + securitySource: client._options.apiKeyHeader, + retryConfig: options?.retries + || client._options.retryConfig + || { strategy: "none" }, + retryCodes: options?.retryCodes || ["429", "500", "502", "503", "504"], + }; + + const requestRes = client._createRequest(context, { + security: requestSecurity, + method: "POST", + baseURL: options?.serverURL, + path: path, + headers: headers, + body: body, + userAgent: client._options.userAgent, + timeoutMs: options?.timeoutMs || client._options.timeoutMs || -1, + }, options); + if (!requestRes.ok) { + return [requestRes, { status: "invalid" }]; + } + const req = requestRes.value; + + const doResult = await client._do(req, { + context, + errorCodes: ["422", "4XX", "5XX"], + retryConfig: context.retryConfig, + retryCodes: context.retryCodes, + }); + if (!doResult.ok) { + return [doResult, { status: "request-error", request: req }]; + } + const response = doResult.value; + + const responseFields = { + HttpMeta: { Response: response, Request: req }, + }; + + const [result] = await M.match< + models.RenderControlTemplateResponse, + | errors.HTTPValidationError + | AgentControlSDKError + | ResponseValidationError + | ConnectionError + | RequestAbortedError + | RequestTimeoutError + | InvalidRequestError + | UnexpectedClientError + | SDKValidationError + >( + M.json(200, models.RenderControlTemplateResponse$inboundSchema), + M.jsonErr(422, errors.HTTPValidationError$inboundSchema), + M.fail("4XX"), + M.fail("5XX"), + )(response, req, { extraFields: responseFields }); + if (!result.ok) { + return [result, { status: "complete", request: req, response }]; + } + + return [result, { status: "complete", request: req, response }]; +} diff --git a/sdks/typescript/src/generated/models/boolean-template-parameter.ts b/sdks/typescript/src/generated/models/boolean-template-parameter.ts new file mode 100644 index 00000000..78cef05d --- /dev/null +++ b/sdks/typescript/src/generated/models/boolean-template-parameter.ts @@ -0,0 +1,103 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +/** + * Boolean template parameter. + */ +export type BooleanTemplateParameter = { + /** + * Optional default value + */ + default?: boolean | null | undefined; + /** + * Optional description of what the parameter controls + */ + description?: string | null | undefined; + /** + * Human-readable parameter label + */ + label: string; + /** + * Whether the caller must provide a value when no default exists + */ + required?: boolean | undefined; + type: "boolean"; + /** + * Optional UI hint for rendering the parameter input + */ + uiHint?: string | null | undefined; +}; + +/** @internal */ +export const BooleanTemplateParameter$inboundSchema: z.ZodMiniType< + BooleanTemplateParameter, + unknown +> = z.pipe( + z.object({ + default: z.optional(z.nullable(types.boolean())), + description: z.optional(z.nullable(types.string())), + label: types.string(), + required: z._default(types.boolean(), true), + type: types.literal("boolean"), + ui_hint: z.optional(z.nullable(types.string())), + }), + z.transform((v) => { + return remap$(v, { + "ui_hint": "uiHint", + }); + }), +); +/** @internal */ +export type BooleanTemplateParameter$Outbound = { + default?: boolean | null | undefined; + description?: string | null | undefined; + label: string; + required: boolean; + type: "boolean"; + ui_hint?: string | null | undefined; +}; + +/** @internal */ +export const BooleanTemplateParameter$outboundSchema: z.ZodMiniType< + BooleanTemplateParameter$Outbound, + BooleanTemplateParameter +> = z.pipe( + z.object({ + default: z.optional(z.nullable(z.boolean())), + description: z.optional(z.nullable(z.string())), + label: z.string(), + required: z._default(z.boolean(), true), + type: z.literal("boolean"), + uiHint: z.optional(z.nullable(z.string())), + }), + z.transform((v) => { + return remap$(v, { + uiHint: "ui_hint", + }); + }), +); + +export function booleanTemplateParameterToJSON( + booleanTemplateParameter: BooleanTemplateParameter, +): string { + return JSON.stringify( + BooleanTemplateParameter$outboundSchema.parse(booleanTemplateParameter), + ); +} +export function booleanTemplateParameterFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => BooleanTemplateParameter$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'BooleanTemplateParameter' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/control-definition-input.ts b/sdks/typescript/src/generated/models/control-definition-input.ts index f385b0d5..bfcf4687 100644 --- a/sdks/typescript/src/generated/models/control-definition-input.ts +++ b/sdks/typescript/src/generated/models/control-definition-input.ts @@ -3,6 +3,7 @@ */ import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; import { ClosedEnum } from "../types/enums.js"; import { ConditionNodeInput, @@ -19,6 +20,16 @@ import { ControlScope$Outbound, ControlScope$outboundSchema, } from "./control-scope.js"; +import { + TemplateDefinitionInput, + TemplateDefinitionInput$Outbound, + TemplateDefinitionInput$outboundSchema, +} from "./template-definition-input.js"; +import { + TemplateValue, + TemplateValue$Outbound, + TemplateValue$outboundSchema, +} from "./template-value.js"; /** * Where this control executes @@ -71,6 +82,14 @@ export type ControlDefinitionInput = { * Tags for categorization */ tags?: Array | undefined; + /** + * Template metadata for template-backed controls + */ + template?: TemplateDefinitionInput | null | undefined; + /** + * Resolved parameter values for template-backed controls + */ + templateValues?: { [k: string]: TemplateValue } | null | undefined; }; /** @internal */ @@ -87,21 +106,34 @@ export type ControlDefinitionInput$Outbound = { execution: string; scope?: ControlScope$Outbound | undefined; tags?: Array | undefined; + template?: TemplateDefinitionInput$Outbound | null | undefined; + template_values?: { [k: string]: TemplateValue$Outbound } | null | undefined; }; /** @internal */ export const ControlDefinitionInput$outboundSchema: z.ZodMiniType< ControlDefinitionInput$Outbound, ControlDefinitionInput -> = z.object({ - action: ControlAction$outboundSchema, - condition: ConditionNodeInput$outboundSchema, - description: z.optional(z.nullable(z.string())), - enabled: z._default(z.boolean(), true), - execution: ControlDefinitionInputExecution$outboundSchema, - scope: z.optional(ControlScope$outboundSchema), - tags: z.optional(z.array(z.string())), -}); +> = z.pipe( + z.object({ + action: ControlAction$outboundSchema, + condition: ConditionNodeInput$outboundSchema, + description: z.optional(z.nullable(z.string())), + enabled: z._default(z.boolean(), true), + execution: ControlDefinitionInputExecution$outboundSchema, + scope: z.optional(ControlScope$outboundSchema), + tags: z.optional(z.array(z.string())), + template: z.optional(z.nullable(TemplateDefinitionInput$outboundSchema)), + templateValues: z.optional( + z.nullable(z.record(z.string(), TemplateValue$outboundSchema)), + ), + }), + z.transform((v) => { + return remap$(v, { + templateValues: "template_values", + }); + }), +); export function controlDefinitionInputToJSON( controlDefinitionInput: ControlDefinitionInput, diff --git a/sdks/typescript/src/generated/models/control-definition-output.ts b/sdks/typescript/src/generated/models/control-definition-output.ts index 8199d5f3..56449cbd 100644 --- a/sdks/typescript/src/generated/models/control-definition-output.ts +++ b/sdks/typescript/src/generated/models/control-definition-output.ts @@ -3,6 +3,7 @@ */ import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; import { safeParse } from "../lib/schemas.js"; import * as openEnums from "../types/enums.js"; import { OpenEnum } from "../types/enums.js"; @@ -18,6 +19,14 @@ import { } from "./control-action.js"; import { ControlScope, ControlScope$inboundSchema } from "./control-scope.js"; import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + TemplateDefinitionOutput, + TemplateDefinitionOutput$inboundSchema, +} from "./template-definition-output.js"; +import { + TemplateValue, + TemplateValue$inboundSchema, +} from "./template-value.js"; /** * Where this control executes @@ -68,6 +77,14 @@ export type ControlDefinitionOutput = { * Tags for categorization */ tags?: Array | undefined; + /** + * Template metadata for template-backed controls + */ + template?: TemplateDefinitionOutput | null | undefined; + /** + * Resolved parameter values for template-backed controls + */ + templateValues?: { [k: string]: TemplateValue } | null | undefined; }; /** @internal */ @@ -78,15 +95,26 @@ export const Execution$inboundSchema: z.ZodMiniType = export const ControlDefinitionOutput$inboundSchema: z.ZodMiniType< ControlDefinitionOutput, unknown -> = z.object({ - action: ControlAction$inboundSchema, - condition: ConditionNodeOutput$inboundSchema, - description: z.optional(z.nullable(types.string())), - enabled: z._default(types.boolean(), true), - execution: Execution$inboundSchema, - scope: types.optional(ControlScope$inboundSchema), - tags: types.optional(z.array(types.string())), -}); +> = z.pipe( + z.object({ + action: ControlAction$inboundSchema, + condition: ConditionNodeOutput$inboundSchema, + description: z.optional(z.nullable(types.string())), + enabled: z._default(types.boolean(), true), + execution: Execution$inboundSchema, + scope: types.optional(ControlScope$inboundSchema), + tags: types.optional(z.array(types.string())), + template: z.optional(z.nullable(TemplateDefinitionOutput$inboundSchema)), + template_values: z.optional( + z.nullable(z.record(z.string(), TemplateValue$inboundSchema)), + ), + }), + z.transform((v) => { + return remap$(v, { + "template_values": "templateValues", + }); + }), +); export function controlDefinitionOutputFromJSON( jsonString: string, diff --git a/sdks/typescript/src/generated/models/control-summary.ts b/sdks/typescript/src/generated/models/control-summary.ts index 7e89a07f..c9c717a4 100644 --- a/sdks/typescript/src/generated/models/control-summary.ts +++ b/sdks/typescript/src/generated/models/control-summary.ts @@ -46,6 +46,10 @@ export type ControlSummary = { * Control tags */ tags?: Array | undefined; + /** + * Whether the control was created from a template + */ + templateBacked: boolean; /** * Agent using this control */ @@ -70,12 +74,14 @@ export const ControlSummary$inboundSchema: z.ZodMiniType< stages: z.optional(z.nullable(z.array(types.string()))), step_types: z.optional(z.nullable(z.array(types.string()))), tags: types.optional(z.array(types.string())), + template_backed: z._default(types.boolean(), false), used_by_agent: z.optional(z.nullable(AgentRef$inboundSchema)), used_by_agents_count: z._default(types.number(), 0), }), z.transform((v) => { return remap$(v, { "step_types": "stepTypes", + "template_backed": "templateBacked", "used_by_agent": "usedByAgent", "used_by_agents_count": "usedByAgentsCount", }); diff --git a/sdks/typescript/src/generated/models/create-control-request.ts b/sdks/typescript/src/generated/models/create-control-request.ts index 83a198ba..147b0354 100644 --- a/sdks/typescript/src/generated/models/create-control-request.ts +++ b/sdks/typescript/src/generated/models/create-control-request.ts @@ -3,31 +3,53 @@ */ import * as z from "zod/v4-mini"; +import { smartUnion } from "../types/smart-union.js"; import { ControlDefinitionInput, ControlDefinitionInput$Outbound, ControlDefinitionInput$outboundSchema, } from "./control-definition-input.js"; +import { + TemplateControlInput, + TemplateControlInput$Outbound, + TemplateControlInput$outboundSchema, +} from "./template-control-input.js"; + +/** + * Control definition to validate and store during creation + */ +export type Data = ControlDefinitionInput | TemplateControlInput; export type CreateControlRequest = { /** - * A control definition to evaluate agent interactions. - * - * @remarks - * - * This model contains only the logic and configuration. - * Identity fields (id, name) are managed by the database. + * Control definition to validate and store during creation */ - data: ControlDefinitionInput; + data: ControlDefinitionInput | TemplateControlInput; /** * Unique control name (letters, numbers, hyphens, underscores) */ name: string; }; +/** @internal */ +export type Data$Outbound = + | ControlDefinitionInput$Outbound + | TemplateControlInput$Outbound; + +/** @internal */ +export const Data$outboundSchema: z.ZodMiniType = + smartUnion([ + ControlDefinitionInput$outboundSchema, + TemplateControlInput$outboundSchema, + ]); + +export function dataToJSON(data: Data): string { + return JSON.stringify(Data$outboundSchema.parse(data)); +} + /** @internal */ export type CreateControlRequest$Outbound = { - data: ControlDefinitionInput$Outbound; + data: ControlDefinitionInput$Outbound | TemplateControlInput$Outbound; name: string; }; @@ -36,7 +58,10 @@ export const CreateControlRequest$outboundSchema: z.ZodMiniType< CreateControlRequest$Outbound, CreateControlRequest > = z.object({ - data: ControlDefinitionInput$outboundSchema, + data: smartUnion([ + ControlDefinitionInput$outboundSchema, + TemplateControlInput$outboundSchema, + ]), name: z.string(), }); diff --git a/sdks/typescript/src/generated/models/enum-template-parameter.ts b/sdks/typescript/src/generated/models/enum-template-parameter.ts new file mode 100644 index 00000000..4f002f6f --- /dev/null +++ b/sdks/typescript/src/generated/models/enum-template-parameter.ts @@ -0,0 +1,112 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +/** + * String enum template parameter. + */ +export type EnumTemplateParameter = { + /** + * Allowed string values for the parameter + */ + allowedValues: Array; + /** + * Optional default value + */ + default?: string | null | undefined; + /** + * Optional description of what the parameter controls + */ + description?: string | null | undefined; + /** + * Human-readable parameter label + */ + label: string; + /** + * Whether the caller must provide a value when no default exists + */ + required?: boolean | undefined; + type: "enum"; + /** + * Optional UI hint for rendering the parameter input + */ + uiHint?: string | null | undefined; +}; + +/** @internal */ +export const EnumTemplateParameter$inboundSchema: z.ZodMiniType< + EnumTemplateParameter, + unknown +> = z.pipe( + z.object({ + allowed_values: z.array(types.string()), + default: z.optional(z.nullable(types.string())), + description: z.optional(z.nullable(types.string())), + label: types.string(), + required: z._default(types.boolean(), true), + type: types.literal("enum"), + ui_hint: z.optional(z.nullable(types.string())), + }), + z.transform((v) => { + return remap$(v, { + "allowed_values": "allowedValues", + "ui_hint": "uiHint", + }); + }), +); +/** @internal */ +export type EnumTemplateParameter$Outbound = { + allowed_values: Array; + default?: string | null | undefined; + description?: string | null | undefined; + label: string; + required: boolean; + type: "enum"; + ui_hint?: string | null | undefined; +}; + +/** @internal */ +export const EnumTemplateParameter$outboundSchema: z.ZodMiniType< + EnumTemplateParameter$Outbound, + EnumTemplateParameter +> = z.pipe( + z.object({ + allowedValues: z.array(z.string()), + default: z.optional(z.nullable(z.string())), + description: z.optional(z.nullable(z.string())), + label: z.string(), + required: z._default(z.boolean(), true), + type: z.literal("enum"), + uiHint: z.optional(z.nullable(z.string())), + }), + z.transform((v) => { + return remap$(v, { + allowedValues: "allowed_values", + uiHint: "ui_hint", + }); + }), +); + +export function enumTemplateParameterToJSON( + enumTemplateParameter: EnumTemplateParameter, +): string { + return JSON.stringify( + EnumTemplateParameter$outboundSchema.parse(enumTemplateParameter), + ); +} +export function enumTemplateParameterFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => EnumTemplateParameter$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'EnumTemplateParameter' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/index.ts b/sdks/typescript/src/generated/models/index.ts index 02085da1..72eeab0a 100644 --- a/sdks/typescript/src/generated/models/index.ts +++ b/sdks/typescript/src/generated/models/index.ts @@ -10,6 +10,7 @@ export * from "./assoc-response.js"; export * from "./auth-mode.js"; export * from "./batch-events-request.js"; export * from "./batch-events-response.js"; +export * from "./boolean-template-parameter.js"; export * from "./condition-node-input.js"; export * from "./condition-node-output.js"; export * from "./config-response.js"; @@ -31,6 +32,7 @@ export * from "./create-policy-request.js"; export * from "./create-policy-response.js"; export * from "./delete-control-response.js"; export * from "./delete-policy-response.js"; +export * from "./enum-template-parameter.js"; export * from "./evaluation-request.js"; export * from "./evaluation-response.js"; export * from "./evaluator-info.js"; @@ -51,6 +53,10 @@ export * from "./init-agent-evaluator-removal.js"; export * from "./init-agent-overwrite-changes.js"; export * from "./init-agent-request.js"; export * from "./init-agent-response.js"; +export * from "./json-value-input.js"; +export * from "./json-value-input1.js"; +export * from "./json-value-output.js"; +export * from "./json-value-output1.js"; export * from "./list-agents-response.js"; export * from "./list-controls-response.js"; export * from "./list-evaluators-response.js"; @@ -61,7 +67,10 @@ export * from "./patch-agent-request.js"; export * from "./patch-agent-response.js"; export * from "./patch-control-request.js"; export * from "./patch-control-response.js"; +export * from "./regex-template-parameter.js"; export * from "./remove-agent-control-response.js"; +export * from "./render-control-template-request.js"; +export * from "./render-control-template-response.js"; export * from "./security.js"; export * from "./set-control-data-request.js"; export * from "./set-control-data-response.js"; @@ -72,6 +81,13 @@ export * from "./steering-context.js"; export * from "./step-key.js"; export * from "./step-schema.js"; export * from "./step.js"; +export * from "./string-list-template-parameter.js"; +export * from "./string-template-parameter.js"; +export * from "./template-control-input.js"; +export * from "./template-definition-input.js"; +export * from "./template-definition-output.js"; +export * from "./template-parameter-definition.js"; +export * from "./template-value.js"; export * from "./timeseries-bucket.js"; export * from "./validate-control-data-request.js"; export * from "./validate-control-data-response.js"; diff --git a/sdks/typescript/src/generated/models/json-value-input.ts b/sdks/typescript/src/generated/models/json-value-input.ts new file mode 100644 index 00000000..4f448073 --- /dev/null +++ b/sdks/typescript/src/generated/models/json-value-input.ts @@ -0,0 +1,40 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { smartUnion } from "../types/smart-union.js"; + +export type JSONValueInput = + | string + | number + | number + | boolean + | Array + | { [k: string]: JSONValueInput | null }; + +/** @internal */ +export type JSONValueInput$Outbound = + | string + | number + | number + | boolean + | Array + | { [k: string]: JSONValueInput$Outbound | null }; + +/** @internal */ +export const JSONValueInput$outboundSchema: z.ZodMiniType< + JSONValueInput$Outbound, + JSONValueInput +> = smartUnion([ + z.string(), + z.int(), + z.number(), + z.boolean(), + z.array(z.nullable(z.lazy(() => JSONValueInput$outboundSchema))), + z.record(z.string(), z.nullable(z.lazy(() => JSONValueInput$outboundSchema))), +]); + +export function jsonValueInputToJSON(jsonValueInput: JSONValueInput): string { + return JSON.stringify(JSONValueInput$outboundSchema.parse(jsonValueInput)); +} diff --git a/sdks/typescript/src/generated/models/json-value-input1.ts b/sdks/typescript/src/generated/models/json-value-input1.ts new file mode 100644 index 00000000..b613f2e4 --- /dev/null +++ b/sdks/typescript/src/generated/models/json-value-input1.ts @@ -0,0 +1,47 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { smartUnion } from "../types/smart-union.js"; +import { + JSONValueInput, + JSONValueInput$Outbound, + JSONValueInput$outboundSchema, +} from "./json-value-input.js"; + +export type JsonValueInput1 = + | string + | number + | number + | boolean + | Array + | { [k: string]: JSONValueInput | null }; + +/** @internal */ +export type JsonValueInput1$Outbound = + | string + | number + | number + | boolean + | Array + | { [k: string]: JSONValueInput$Outbound | null }; + +/** @internal */ +export const JsonValueInput1$outboundSchema: z.ZodMiniType< + JsonValueInput1$Outbound, + JsonValueInput1 +> = smartUnion([ + z.string(), + z.int(), + z.number(), + z.boolean(), + z.array(z.nullable(JSONValueInput$outboundSchema)), + z.record(z.string(), z.nullable(z.lazy(() => JSONValueInput$outboundSchema))), +]); + +export function jsonValueInput1ToJSON( + jsonValueInput1: JsonValueInput1, +): string { + return JSON.stringify(JsonValueInput1$outboundSchema.parse(jsonValueInput1)); +} diff --git a/sdks/typescript/src/generated/models/json-value-output.ts b/sdks/typescript/src/generated/models/json-value-output.ts new file mode 100644 index 00000000..f50e2790 --- /dev/null +++ b/sdks/typescript/src/generated/models/json-value-output.ts @@ -0,0 +1,44 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { smartUnion } from "../types/smart-union.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +export type JSONValueOutput = + | string + | number + | number + | boolean + | Array + | { [k: string]: JSONValueOutput | null }; + +/** @internal */ +export const JSONValueOutput$inboundSchema: z.ZodMiniType< + JSONValueOutput, + unknown +> = smartUnion([ + types.string(), + types.number(), + types.number(), + types.boolean(), + z.array(types.nullable(z.lazy(() => JSONValueOutput$inboundSchema))), + z.record( + z.string(), + types.nullable(z.lazy(() => JSONValueOutput$inboundSchema)), + ), +]); + +export function jsonValueOutputFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => JSONValueOutput$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'JSONValueOutput' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/json-value-output1.ts b/sdks/typescript/src/generated/models/json-value-output1.ts new file mode 100644 index 00000000..877520c3 --- /dev/null +++ b/sdks/typescript/src/generated/models/json-value-output1.ts @@ -0,0 +1,48 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { smartUnion } from "../types/smart-union.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + JSONValueOutput, + JSONValueOutput$inboundSchema, +} from "./json-value-output.js"; + +export type JsonValueOutput1 = + | string + | number + | number + | boolean + | Array + | { [k: string]: JSONValueOutput | null }; + +/** @internal */ +export const JsonValueOutput1$inboundSchema: z.ZodMiniType< + JsonValueOutput1, + unknown +> = smartUnion([ + types.string(), + types.number(), + types.number(), + types.boolean(), + z.array(types.nullable(JSONValueOutput$inboundSchema)), + z.record( + z.string(), + types.nullable(z.lazy(() => JSONValueOutput$inboundSchema)), + ), +]); + +export function jsonValueOutput1FromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => JsonValueOutput1$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'JsonValueOutput1' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/operations/list-controls-api-v1-controls-get.ts b/sdks/typescript/src/generated/models/operations/list-controls-api-v1-controls-get.ts index c040fdf8..7f19162e 100644 --- a/sdks/typescript/src/generated/models/operations/list-controls-api-v1-controls-get.ts +++ b/sdks/typescript/src/generated/models/operations/list-controls-api-v1-controls-get.ts @@ -19,6 +19,10 @@ export type ListControlsApiV1ControlsGetRequest = { * Filter by enabled status */ enabled?: boolean | null | undefined; + /** + * Filter by whether the control is template-backed + */ + templateBacked?: boolean | null | undefined; /** * Filter by step type (built-ins: 'tool', 'llm') */ @@ -43,6 +47,7 @@ export type ListControlsApiV1ControlsGetRequest$Outbound = { limit: number; name?: string | null | undefined; enabled?: boolean | null | undefined; + template_backed?: boolean | null | undefined; step_type?: string | null | undefined; stage?: string | null | undefined; execution?: string | null | undefined; @@ -59,6 +64,7 @@ export const ListControlsApiV1ControlsGetRequest$outboundSchema: z.ZodMiniType< limit: z._default(z.int(), 20), name: z.optional(z.nullable(z.string())), enabled: z.optional(z.nullable(z.boolean())), + templateBacked: z.optional(z.nullable(z.boolean())), stepType: z.optional(z.nullable(z.string())), stage: z.optional(z.nullable(z.string())), execution: z.optional(z.nullable(z.string())), @@ -66,6 +72,7 @@ export const ListControlsApiV1ControlsGetRequest$outboundSchema: z.ZodMiniType< }), z.transform((v) => { return remap$(v, { + templateBacked: "template_backed", stepType: "step_type", }); }), diff --git a/sdks/typescript/src/generated/models/regex-template-parameter.ts b/sdks/typescript/src/generated/models/regex-template-parameter.ts new file mode 100644 index 00000000..eac4ce3d --- /dev/null +++ b/sdks/typescript/src/generated/models/regex-template-parameter.ts @@ -0,0 +1,110 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +/** + * RE2 regex template parameter. + */ +export type RegexTemplateParameter = { + /** + * Optional default regex pattern + */ + default?: string | null | undefined; + /** + * Optional description of what the parameter controls + */ + description?: string | null | undefined; + /** + * Human-readable parameter label + */ + label: string; + /** + * Optional placeholder regex + */ + placeholder?: string | null | undefined; + /** + * Whether the caller must provide a value when no default exists + */ + required?: boolean | undefined; + type: "regex_re2"; + /** + * Optional UI hint for rendering the parameter input + */ + uiHint?: string | null | undefined; +}; + +/** @internal */ +export const RegexTemplateParameter$inboundSchema: z.ZodMiniType< + RegexTemplateParameter, + unknown +> = z.pipe( + z.object({ + default: z.optional(z.nullable(types.string())), + description: z.optional(z.nullable(types.string())), + label: types.string(), + placeholder: z.optional(z.nullable(types.string())), + required: z._default(types.boolean(), true), + type: types.literal("regex_re2"), + ui_hint: z.optional(z.nullable(types.string())), + }), + z.transform((v) => { + return remap$(v, { + "ui_hint": "uiHint", + }); + }), +); +/** @internal */ +export type RegexTemplateParameter$Outbound = { + default?: string | null | undefined; + description?: string | null | undefined; + label: string; + placeholder?: string | null | undefined; + required: boolean; + type: "regex_re2"; + ui_hint?: string | null | undefined; +}; + +/** @internal */ +export const RegexTemplateParameter$outboundSchema: z.ZodMiniType< + RegexTemplateParameter$Outbound, + RegexTemplateParameter +> = z.pipe( + z.object({ + default: z.optional(z.nullable(z.string())), + description: z.optional(z.nullable(z.string())), + label: z.string(), + placeholder: z.optional(z.nullable(z.string())), + required: z._default(z.boolean(), true), + type: z.literal("regex_re2"), + uiHint: z.optional(z.nullable(z.string())), + }), + z.transform((v) => { + return remap$(v, { + uiHint: "ui_hint", + }); + }), +); + +export function regexTemplateParameterToJSON( + regexTemplateParameter: RegexTemplateParameter, +): string { + return JSON.stringify( + RegexTemplateParameter$outboundSchema.parse(regexTemplateParameter), + ); +} +export function regexTemplateParameterFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => RegexTemplateParameter$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'RegexTemplateParameter' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/render-control-template-request.ts b/sdks/typescript/src/generated/models/render-control-template-request.ts new file mode 100644 index 00000000..03211681 --- /dev/null +++ b/sdks/typescript/src/generated/models/render-control-template-request.ts @@ -0,0 +1,64 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { + TemplateDefinitionInput, + TemplateDefinitionInput$Outbound, + TemplateDefinitionInput$outboundSchema, +} from "./template-definition-input.js"; +import { + TemplateValue, + TemplateValue$Outbound, + TemplateValue$outboundSchema, +} from "./template-value.js"; + +/** + * Request to render a template-backed control without persisting it. + */ +export type RenderControlTemplateRequest = { + /** + * Reusable template with typed parameters and a JSON definition template. + */ + template: TemplateDefinitionInput; + /** + * Template parameter values used during rendering + */ + templateValues?: { [k: string]: TemplateValue } | undefined; +}; + +/** @internal */ +export type RenderControlTemplateRequest$Outbound = { + template: TemplateDefinitionInput$Outbound; + template_values?: { [k: string]: TemplateValue$Outbound } | undefined; +}; + +/** @internal */ +export const RenderControlTemplateRequest$outboundSchema: z.ZodMiniType< + RenderControlTemplateRequest$Outbound, + RenderControlTemplateRequest +> = z.pipe( + z.object({ + template: TemplateDefinitionInput$outboundSchema, + templateValues: z.optional( + z.record(z.string(), TemplateValue$outboundSchema), + ), + }), + z.transform((v) => { + return remap$(v, { + templateValues: "template_values", + }); + }), +); + +export function renderControlTemplateRequestToJSON( + renderControlTemplateRequest: RenderControlTemplateRequest, +): string { + return JSON.stringify( + RenderControlTemplateRequest$outboundSchema.parse( + renderControlTemplateRequest, + ), + ); +} diff --git a/sdks/typescript/src/generated/models/render-control-template-response.ts b/sdks/typescript/src/generated/models/render-control-template-response.ts new file mode 100644 index 00000000..012a966e --- /dev/null +++ b/sdks/typescript/src/generated/models/render-control-template-response.ts @@ -0,0 +1,45 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import { + ControlDefinitionOutput, + ControlDefinitionOutput$inboundSchema, +} from "./control-definition-output.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +/** + * Rendered template preview response. + */ +export type RenderControlTemplateResponse = { + /** + * A control definition to evaluate agent interactions. + * + * @remarks + * + * This model contains only the logic and configuration. + * Identity fields (id, name) are managed by the database. + */ + control: ControlDefinitionOutput; +}; + +/** @internal */ +export const RenderControlTemplateResponse$inboundSchema: z.ZodMiniType< + RenderControlTemplateResponse, + unknown +> = z.object({ + control: ControlDefinitionOutput$inboundSchema, +}); + +export function renderControlTemplateResponseFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => RenderControlTemplateResponse$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'RenderControlTemplateResponse' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/set-control-data-request.ts b/sdks/typescript/src/generated/models/set-control-data-request.ts index 4c1e2bb5..4e434582 100644 --- a/sdks/typescript/src/generated/models/set-control-data-request.ts +++ b/sdks/typescript/src/generated/models/set-control-data-request.ts @@ -3,30 +3,60 @@ */ import * as z from "zod/v4-mini"; +import { smartUnion } from "../types/smart-union.js"; import { ControlDefinitionInput, ControlDefinitionInput$Outbound, ControlDefinitionInput$outboundSchema, } from "./control-definition-input.js"; +import { + TemplateControlInput, + TemplateControlInput$Outbound, + TemplateControlInput$outboundSchema, +} from "./template-control-input.js"; + +/** + * Control configuration data (replaces existing) + */ +export type SetControlDataRequestData = + | ControlDefinitionInput + | TemplateControlInput; /** * Request to update control configuration data. */ export type SetControlDataRequest = { /** - * A control definition to evaluate agent interactions. - * - * @remarks - * - * This model contains only the logic and configuration. - * Identity fields (id, name) are managed by the database. + * Control configuration data (replaces existing) */ - data: ControlDefinitionInput; + data: ControlDefinitionInput | TemplateControlInput; }; +/** @internal */ +export type SetControlDataRequestData$Outbound = + | ControlDefinitionInput$Outbound + | TemplateControlInput$Outbound; + +/** @internal */ +export const SetControlDataRequestData$outboundSchema: z.ZodMiniType< + SetControlDataRequestData$Outbound, + SetControlDataRequestData +> = smartUnion([ + ControlDefinitionInput$outboundSchema, + TemplateControlInput$outboundSchema, +]); + +export function setControlDataRequestDataToJSON( + setControlDataRequestData: SetControlDataRequestData, +): string { + return JSON.stringify( + SetControlDataRequestData$outboundSchema.parse(setControlDataRequestData), + ); +} + /** @internal */ export type SetControlDataRequest$Outbound = { - data: ControlDefinitionInput$Outbound; + data: ControlDefinitionInput$Outbound | TemplateControlInput$Outbound; }; /** @internal */ @@ -34,7 +64,10 @@ export const SetControlDataRequest$outboundSchema: z.ZodMiniType< SetControlDataRequest$Outbound, SetControlDataRequest > = z.object({ - data: ControlDefinitionInput$outboundSchema, + data: smartUnion([ + ControlDefinitionInput$outboundSchema, + TemplateControlInput$outboundSchema, + ]), }); export function setControlDataRequestToJSON( diff --git a/sdks/typescript/src/generated/models/step.ts b/sdks/typescript/src/generated/models/step.ts index 132cf9c9..8c3d4468 100644 --- a/sdks/typescript/src/generated/models/step.ts +++ b/sdks/typescript/src/generated/models/step.ts @@ -3,6 +3,11 @@ */ import * as z from "zod/v4-mini"; +import { + JSONValueInput, + JSONValueInput$Outbound, + JSONValueInput$outboundSchema, +} from "./json-value-input.js"; /** * Runtime payload for an agent step invocation. @@ -11,11 +16,8 @@ export type Step = { /** * Optional context (conversation history, metadata, etc.) */ - context?: { [k: string]: any } | null | undefined; - /** - * Any JSON value - */ - input: any; + context?: { [k: string]: JSONValueInput | null } | null | undefined; + input: JSONValueInput | null; /** * Step name (tool name or model/chain id) */ @@ -23,7 +25,7 @@ export type Step = { /** * Output content for this step (None for pre-checks) */ - output?: any | null | undefined; + output?: JSONValueInput | null | undefined; /** * Step type (e.g., 'tool', 'llm') */ @@ -32,20 +34,24 @@ export type Step = { /** @internal */ export type Step$Outbound = { - context?: { [k: string]: any } | null | undefined; - input: any; + context?: { [k: string]: JSONValueInput$Outbound | null } | null | undefined; + input: JSONValueInput$Outbound | null; name: string; - output?: any | null | undefined; + output?: JSONValueInput$Outbound | null | undefined; type: string; }; /** @internal */ export const Step$outboundSchema: z.ZodMiniType = z.object( { - context: z.optional(z.nullable(z.record(z.string(), z.any()))), - input: z.any(), + context: z.optional( + z.nullable( + z.record(z.string(), z.nullable(JSONValueInput$outboundSchema)), + ), + ), + input: z.nullable(JSONValueInput$outboundSchema), name: z.string(), - output: z.optional(z.nullable(z.any())), + output: z.optional(z.nullable(JSONValueInput$outboundSchema)), type: z.string(), }, ); diff --git a/sdks/typescript/src/generated/models/string-list-template-parameter.ts b/sdks/typescript/src/generated/models/string-list-template-parameter.ts new file mode 100644 index 00000000..d21ddc37 --- /dev/null +++ b/sdks/typescript/src/generated/models/string-list-template-parameter.ts @@ -0,0 +1,112 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +/** + * List-of-strings template parameter. + */ +export type StringListTemplateParameter = { + /** + * Optional default value + */ + default?: Array | null | undefined; + /** + * Optional description of what the parameter controls + */ + description?: string | null | undefined; + /** + * Human-readable parameter label + */ + label: string; + /** + * Optional placeholder/example list + */ + placeholder?: Array | null | undefined; + /** + * Whether the caller must provide a value when no default exists + */ + required?: boolean | undefined; + type: "string_list"; + /** + * Optional UI hint for rendering the parameter input + */ + uiHint?: string | null | undefined; +}; + +/** @internal */ +export const StringListTemplateParameter$inboundSchema: z.ZodMiniType< + StringListTemplateParameter, + unknown +> = z.pipe( + z.object({ + default: z.optional(z.nullable(z.array(types.string()))), + description: z.optional(z.nullable(types.string())), + label: types.string(), + placeholder: z.optional(z.nullable(z.array(types.string()))), + required: z._default(types.boolean(), true), + type: types.literal("string_list"), + ui_hint: z.optional(z.nullable(types.string())), + }), + z.transform((v) => { + return remap$(v, { + "ui_hint": "uiHint", + }); + }), +); +/** @internal */ +export type StringListTemplateParameter$Outbound = { + default?: Array | null | undefined; + description?: string | null | undefined; + label: string; + placeholder?: Array | null | undefined; + required: boolean; + type: "string_list"; + ui_hint?: string | null | undefined; +}; + +/** @internal */ +export const StringListTemplateParameter$outboundSchema: z.ZodMiniType< + StringListTemplateParameter$Outbound, + StringListTemplateParameter +> = z.pipe( + z.object({ + default: z.optional(z.nullable(z.array(z.string()))), + description: z.optional(z.nullable(z.string())), + label: z.string(), + placeholder: z.optional(z.nullable(z.array(z.string()))), + required: z._default(z.boolean(), true), + type: z.literal("string_list"), + uiHint: z.optional(z.nullable(z.string())), + }), + z.transform((v) => { + return remap$(v, { + uiHint: "ui_hint", + }); + }), +); + +export function stringListTemplateParameterToJSON( + stringListTemplateParameter: StringListTemplateParameter, +): string { + return JSON.stringify( + StringListTemplateParameter$outboundSchema.parse( + stringListTemplateParameter, + ), + ); +} +export function stringListTemplateParameterFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => StringListTemplateParameter$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'StringListTemplateParameter' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/string-template-parameter.ts b/sdks/typescript/src/generated/models/string-template-parameter.ts new file mode 100644 index 00000000..da9787f1 --- /dev/null +++ b/sdks/typescript/src/generated/models/string-template-parameter.ts @@ -0,0 +1,110 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +/** + * String-valued template parameter. + */ +export type StringTemplateParameter = { + /** + * Optional default value + */ + default?: string | null | undefined; + /** + * Optional description of what the parameter controls + */ + description?: string | null | undefined; + /** + * Human-readable parameter label + */ + label: string; + /** + * Optional placeholder text + */ + placeholder?: string | null | undefined; + /** + * Whether the caller must provide a value when no default exists + */ + required?: boolean | undefined; + type: "string"; + /** + * Optional UI hint for rendering the parameter input + */ + uiHint?: string | null | undefined; +}; + +/** @internal */ +export const StringTemplateParameter$inboundSchema: z.ZodMiniType< + StringTemplateParameter, + unknown +> = z.pipe( + z.object({ + default: z.optional(z.nullable(types.string())), + description: z.optional(z.nullable(types.string())), + label: types.string(), + placeholder: z.optional(z.nullable(types.string())), + required: z._default(types.boolean(), true), + type: types.literal("string"), + ui_hint: z.optional(z.nullable(types.string())), + }), + z.transform((v) => { + return remap$(v, { + "ui_hint": "uiHint", + }); + }), +); +/** @internal */ +export type StringTemplateParameter$Outbound = { + default?: string | null | undefined; + description?: string | null | undefined; + label: string; + placeholder?: string | null | undefined; + required: boolean; + type: "string"; + ui_hint?: string | null | undefined; +}; + +/** @internal */ +export const StringTemplateParameter$outboundSchema: z.ZodMiniType< + StringTemplateParameter$Outbound, + StringTemplateParameter +> = z.pipe( + z.object({ + default: z.optional(z.nullable(z.string())), + description: z.optional(z.nullable(z.string())), + label: z.string(), + placeholder: z.optional(z.nullable(z.string())), + required: z._default(z.boolean(), true), + type: z.literal("string"), + uiHint: z.optional(z.nullable(z.string())), + }), + z.transform((v) => { + return remap$(v, { + uiHint: "ui_hint", + }); + }), +); + +export function stringTemplateParameterToJSON( + stringTemplateParameter: StringTemplateParameter, +): string { + return JSON.stringify( + StringTemplateParameter$outboundSchema.parse(stringTemplateParameter), + ); +} +export function stringTemplateParameterFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => StringTemplateParameter$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'StringTemplateParameter' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/template-control-input.ts b/sdks/typescript/src/generated/models/template-control-input.ts new file mode 100644 index 00000000..2bf49fed --- /dev/null +++ b/sdks/typescript/src/generated/models/template-control-input.ts @@ -0,0 +1,62 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { + TemplateDefinitionInput, + TemplateDefinitionInput$Outbound, + TemplateDefinitionInput$outboundSchema, +} from "./template-definition-input.js"; +import { + TemplateValue, + TemplateValue$Outbound, + TemplateValue$outboundSchema, +} from "./template-value.js"; + +/** + * Template-backed input payload for control create/update requests. + */ +export type TemplateControlInput = { + /** + * Reusable template with typed parameters and a JSON definition template. + */ + template: TemplateDefinitionInput; + /** + * Template parameter values keyed by parameter name + */ + templateValues?: { [k: string]: TemplateValue } | undefined; +}; + +/** @internal */ +export type TemplateControlInput$Outbound = { + template: TemplateDefinitionInput$Outbound; + template_values?: { [k: string]: TemplateValue$Outbound } | undefined; +}; + +/** @internal */ +export const TemplateControlInput$outboundSchema: z.ZodMiniType< + TemplateControlInput$Outbound, + TemplateControlInput +> = z.pipe( + z.object({ + template: TemplateDefinitionInput$outboundSchema, + templateValues: z.optional( + z.record(z.string(), TemplateValue$outboundSchema), + ), + }), + z.transform((v) => { + return remap$(v, { + templateValues: "template_values", + }); + }), +); + +export function templateControlInputToJSON( + templateControlInput: TemplateControlInput, +): string { + return JSON.stringify( + TemplateControlInput$outboundSchema.parse(templateControlInput), + ); +} diff --git a/sdks/typescript/src/generated/models/template-definition-input.ts b/sdks/typescript/src/generated/models/template-definition-input.ts new file mode 100644 index 00000000..61e40755 --- /dev/null +++ b/sdks/typescript/src/generated/models/template-definition-input.ts @@ -0,0 +1,67 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { + JsonValueInput1, + JsonValueInput1$Outbound, + JsonValueInput1$outboundSchema, +} from "./json-value-input1.js"; +import { + TemplateParameterDefinition, + TemplateParameterDefinition$Outbound, + TemplateParameterDefinition$outboundSchema, +} from "./template-parameter-definition.js"; + +/** + * Reusable template with typed parameters and a JSON definition template. + */ +export type TemplateDefinitionInput = { + definitionTemplate: JsonValueInput1 | null; + /** + * Metadata describing the template itself + */ + description?: string | null | undefined; + /** + * Typed parameter definitions keyed by parameter name + */ + parameters?: { [k: string]: TemplateParameterDefinition } | undefined; +}; + +/** @internal */ +export type TemplateDefinitionInput$Outbound = { + definition_template: JsonValueInput1$Outbound | null; + description?: string | null | undefined; + parameters?: + | { [k: string]: TemplateParameterDefinition$Outbound } + | undefined; +}; + +/** @internal */ +export const TemplateDefinitionInput$outboundSchema: z.ZodMiniType< + TemplateDefinitionInput$Outbound, + TemplateDefinitionInput +> = z.pipe( + z.object({ + definitionTemplate: z.nullable(JsonValueInput1$outboundSchema), + description: z.optional(z.nullable(z.string())), + parameters: z.optional( + z.record(z.string(), TemplateParameterDefinition$outboundSchema), + ), + }), + z.transform((v) => { + return remap$(v, { + definitionTemplate: "definition_template", + }); + }), +); + +export function templateDefinitionInputToJSON( + templateDefinitionInput: TemplateDefinitionInput, +): string { + return JSON.stringify( + TemplateDefinitionInput$outboundSchema.parse(templateDefinitionInput), + ); +} diff --git a/sdks/typescript/src/generated/models/template-definition-output.ts b/sdks/typescript/src/generated/models/template-definition-output.ts new file mode 100644 index 00000000..b246dd7d --- /dev/null +++ b/sdks/typescript/src/generated/models/template-definition-output.ts @@ -0,0 +1,62 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + JsonValueOutput1, + JsonValueOutput1$inboundSchema, +} from "./json-value-output1.js"; +import { + TemplateParameterDefinition, + TemplateParameterDefinition$inboundSchema, +} from "./template-parameter-definition.js"; + +/** + * Reusable template with typed parameters and a JSON definition template. + */ +export type TemplateDefinitionOutput = { + definitionTemplate: JsonValueOutput1 | null; + /** + * Metadata describing the template itself + */ + description?: string | null | undefined; + /** + * Typed parameter definitions keyed by parameter name + */ + parameters?: { [k: string]: TemplateParameterDefinition } | undefined; +}; + +/** @internal */ +export const TemplateDefinitionOutput$inboundSchema: z.ZodMiniType< + TemplateDefinitionOutput, + unknown +> = z.pipe( + z.object({ + definition_template: types.nullable(JsonValueOutput1$inboundSchema), + description: z.optional(z.nullable(types.string())), + parameters: types.optional( + z.record(z.string(), TemplateParameterDefinition$inboundSchema), + ), + }), + z.transform((v) => { + return remap$(v, { + "definition_template": "definitionTemplate", + }); + }), +); + +export function templateDefinitionOutputFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => TemplateDefinitionOutput$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'TemplateDefinitionOutput' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/template-parameter-definition.ts b/sdks/typescript/src/generated/models/template-parameter-definition.ts new file mode 100644 index 00000000..f095caaf --- /dev/null +++ b/sdks/typescript/src/generated/models/template-parameter-definition.ts @@ -0,0 +1,98 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { safeParse } from "../lib/schemas.js"; +import * as discriminatedUnionTypes from "../types/discriminated-union.js"; +import { discriminatedUnion } from "../types/discriminated-union.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import { + BooleanTemplateParameter, + BooleanTemplateParameter$inboundSchema, + BooleanTemplateParameter$Outbound, + BooleanTemplateParameter$outboundSchema, +} from "./boolean-template-parameter.js"; +import { + EnumTemplateParameter, + EnumTemplateParameter$inboundSchema, + EnumTemplateParameter$Outbound, + EnumTemplateParameter$outboundSchema, +} from "./enum-template-parameter.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + RegexTemplateParameter, + RegexTemplateParameter$inboundSchema, + RegexTemplateParameter$Outbound, + RegexTemplateParameter$outboundSchema, +} from "./regex-template-parameter.js"; +import { + StringListTemplateParameter, + StringListTemplateParameter$inboundSchema, + StringListTemplateParameter$Outbound, + StringListTemplateParameter$outboundSchema, +} from "./string-list-template-parameter.js"; +import { + StringTemplateParameter, + StringTemplateParameter$inboundSchema, + StringTemplateParameter$Outbound, + StringTemplateParameter$outboundSchema, +} from "./string-template-parameter.js"; + +export type TemplateParameterDefinition = + | BooleanTemplateParameter + | EnumTemplateParameter + | RegexTemplateParameter + | StringTemplateParameter + | StringListTemplateParameter + | discriminatedUnionTypes.Unknown<"type">; + +/** @internal */ +export const TemplateParameterDefinition$inboundSchema: z.ZodMiniType< + TemplateParameterDefinition, + unknown +> = discriminatedUnion("type", { + boolean: BooleanTemplateParameter$inboundSchema, + enum: EnumTemplateParameter$inboundSchema, + regex_re2: RegexTemplateParameter$inboundSchema, + string: StringTemplateParameter$inboundSchema, + string_list: StringListTemplateParameter$inboundSchema, +}); +/** @internal */ +export type TemplateParameterDefinition$Outbound = + | BooleanTemplateParameter$Outbound + | EnumTemplateParameter$Outbound + | RegexTemplateParameter$Outbound + | StringTemplateParameter$Outbound + | StringListTemplateParameter$Outbound; + +/** @internal */ +export const TemplateParameterDefinition$outboundSchema: z.ZodMiniType< + TemplateParameterDefinition$Outbound, + TemplateParameterDefinition +> = z.union([ + BooleanTemplateParameter$outboundSchema, + EnumTemplateParameter$outboundSchema, + RegexTemplateParameter$outboundSchema, + StringTemplateParameter$outboundSchema, + StringListTemplateParameter$outboundSchema, +]); + +export function templateParameterDefinitionToJSON( + templateParameterDefinition: TemplateParameterDefinition, +): string { + return JSON.stringify( + TemplateParameterDefinition$outboundSchema.parse( + templateParameterDefinition, + ), + ); +} +export function templateParameterDefinitionFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => TemplateParameterDefinition$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'TemplateParameterDefinition' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/template-value.ts b/sdks/typescript/src/generated/models/template-value.ts new file mode 100644 index 00000000..fede1821 --- /dev/null +++ b/sdks/typescript/src/generated/models/template-value.ts @@ -0,0 +1,39 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { smartUnion } from "../types/smart-union.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; + +export type TemplateValue = string | boolean | Array; + +/** @internal */ +export const TemplateValue$inboundSchema: z.ZodMiniType< + TemplateValue, + unknown +> = smartUnion([types.string(), types.boolean(), z.array(types.string())]); +/** @internal */ +export type TemplateValue$Outbound = string | boolean | Array; + +/** @internal */ +export const TemplateValue$outboundSchema: z.ZodMiniType< + TemplateValue$Outbound, + TemplateValue +> = smartUnion([z.string(), z.boolean(), z.array(z.string())]); + +export function templateValueToJSON(templateValue: TemplateValue): string { + return JSON.stringify(TemplateValue$outboundSchema.parse(templateValue)); +} +export function templateValueFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => TemplateValue$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'TemplateValue' from JSON`, + ); +} diff --git a/sdks/typescript/src/generated/models/validate-control-data-request.ts b/sdks/typescript/src/generated/models/validate-control-data-request.ts index 8f340622..17dea583 100644 --- a/sdks/typescript/src/generated/models/validate-control-data-request.ts +++ b/sdks/typescript/src/generated/models/validate-control-data-request.ts @@ -3,30 +3,62 @@ */ import * as z from "zod/v4-mini"; +import { smartUnion } from "../types/smart-union.js"; import { ControlDefinitionInput, ControlDefinitionInput$Outbound, ControlDefinitionInput$outboundSchema, } from "./control-definition-input.js"; +import { + TemplateControlInput, + TemplateControlInput$Outbound, + TemplateControlInput$outboundSchema, +} from "./template-control-input.js"; + +/** + * Control configuration data to validate + */ +export type ValidateControlDataRequestData = + | ControlDefinitionInput + | TemplateControlInput; /** * Request to validate control configuration data without saving. */ export type ValidateControlDataRequest = { /** - * A control definition to evaluate agent interactions. - * - * @remarks - * - * This model contains only the logic and configuration. - * Identity fields (id, name) are managed by the database. + * Control configuration data to validate */ - data: ControlDefinitionInput; + data: ControlDefinitionInput | TemplateControlInput; }; +/** @internal */ +export type ValidateControlDataRequestData$Outbound = + | ControlDefinitionInput$Outbound + | TemplateControlInput$Outbound; + +/** @internal */ +export const ValidateControlDataRequestData$outboundSchema: z.ZodMiniType< + ValidateControlDataRequestData$Outbound, + ValidateControlDataRequestData +> = smartUnion([ + ControlDefinitionInput$outboundSchema, + TemplateControlInput$outboundSchema, +]); + +export function validateControlDataRequestDataToJSON( + validateControlDataRequestData: ValidateControlDataRequestData, +): string { + return JSON.stringify( + ValidateControlDataRequestData$outboundSchema.parse( + validateControlDataRequestData, + ), + ); +} + /** @internal */ export type ValidateControlDataRequest$Outbound = { - data: ControlDefinitionInput$Outbound; + data: ControlDefinitionInput$Outbound | TemplateControlInput$Outbound; }; /** @internal */ @@ -34,7 +66,10 @@ export const ValidateControlDataRequest$outboundSchema: z.ZodMiniType< ValidateControlDataRequest$Outbound, ValidateControlDataRequest > = z.object({ - data: ControlDefinitionInput$outboundSchema, + data: smartUnion([ + ControlDefinitionInput$outboundSchema, + TemplateControlInput$outboundSchema, + ]), }); export function validateControlDataRequestToJSON( diff --git a/sdks/typescript/src/generated/sdk/controls.ts b/sdks/typescript/src/generated/sdk/controls.ts index 67583335..3cc982fe 100644 --- a/sdks/typescript/src/generated/sdk/controls.ts +++ b/sdks/typescript/src/generated/sdk/controls.ts @@ -7,6 +7,7 @@ import { controlsDelete } from "../funcs/controls-delete.js"; import { controlsGetData } from "../funcs/controls-get-data.js"; import { controlsGet } from "../funcs/controls-get.js"; import { controlsList } from "../funcs/controls-list.js"; +import { controlsRenderTemplate } from "../funcs/controls-render-template.js"; import { controlsUpdateData } from "../funcs/controls-update-data.js"; import { controlsUpdateMetadata } from "../funcs/controls-update-metadata.js"; import { controlsValidateData } from "../funcs/controls-validate-data.js"; @@ -16,6 +17,23 @@ import * as operations from "../models/operations/index.js"; import { unwrapAsync } from "../types/fp.js"; export class Controls extends ClientSDK { + /** + * Render a control template preview + * + * @remarks + * Render a template-backed control without persisting it. + */ + async renderTemplate( + request: models.RenderControlTemplateRequest, + options?: RequestOptions, + ): Promise { + return unwrapAsync(controlsRenderTemplate( + this, + request, + options, + )); + } + /** * List all controls * @@ -29,6 +47,7 @@ export class Controls extends ClientSDK { * limit: Maximum number of controls to return (default 20, max 100) * name: Optional filter by name (partial, case-insensitive match) * enabled: Optional filter by enabled status + * template_backed: Optional filter by whether the control is template-backed * step_type: Optional filter by step type (built-ins: 'tool', 'llm') * stage: Optional filter by stage ('pre' or 'post') * execution: Optional filter by execution ('server' or 'sdk') diff --git a/sdks/typescript/src/generated/types/discriminated-union.ts b/sdks/typescript/src/generated/types/discriminated-union.ts new file mode 100644 index 00000000..f41a6ac1 --- /dev/null +++ b/sdks/typescript/src/generated/types/discriminated-union.ts @@ -0,0 +1,101 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { startCountingDefaultToZeroValue } from "./default-to-zero-value.js"; +import { startCountingUnrecognized } from "./unrecognized.js"; + +const UNKNOWN = Symbol("UNKNOWN"); + +export type Unknown = + & { + [K in Discriminator]: UnknownValue; + } + & { + raw: unknown; + isUnknown: true; + }; + +export function isUnknown( + value: unknown, +): value is Unknown { + return typeof value === "object" && value !== null && UNKNOWN in value; +} + +/** + * Forward-compatible discriminated union parser. + * + * If the input does not match one of the predefined options, it will be + * captured and available as `{ raw: , [discriminator]: "UNKNOWN", isUnknown: true }`. + * + * @param inputPropertyName - The discriminator property name in the input payload + * @param options - Map of discriminator values to their corresponding schemas + * @param opts - Optional configuration object + * @param opts.unknownValue - The value to use for the discriminator when the input is unknown (default: "UNKNOWN") + * @param opts.outputPropertyName - Output property name if the sanitized (camelCase) property name differs from inputPropertyName + */ +export function discriminatedUnion< + InputDiscriminator extends string, + TOptions extends Readonly>, + UnknownValue extends string = "UNKNOWN", + OutputDiscriminator extends string = InputDiscriminator, +>( + inputPropertyName: InputDiscriminator, + options: TOptions, + opts: { + unknownValue?: UnknownValue; + outputPropertyName?: OutputDiscriminator; + } = {}, +): z.ZodMiniType< + | z.output + | Unknown, + unknown +> { + const { unknownValue = "UNKNOWN" as UnknownValue, outputPropertyName } = opts; + return z.pipe( + z.unknown(), + z.transform((input) => { + const fallback = Object.defineProperties( + { + raw: input, + [outputPropertyName ?? inputPropertyName]: unknownValue, + isUnknown: true as const, + }, + { [UNKNOWN]: { value: true, enumerable: false, configurable: false } }, + ); + + const isObject = typeof input === "object" && input !== null; + if (!isObject) return fallback; + + const discriminator = input[inputPropertyName as keyof typeof input]; + if (typeof discriminator !== "string") return fallback; + if (!(discriminator in options)) return fallback; + + const schema = options[discriminator]; + if (!schema) return fallback; + + // Start counters before parsing to track nested unrecognized/zeroDefault values + const unrecognizedCtr = startCountingUnrecognized(); + const zeroDefaultCtr = startCountingDefaultToZeroValue(); + + const result = z.safeParse(schema, input); + if (!result.success) { + // Parse failed - don't propagate any counts from the failed attempt + unrecognizedCtr.end(0); + zeroDefaultCtr.end(0); + return fallback; + } + + // Parse succeeded - propagate the actual counts + unrecognizedCtr.end(); + zeroDefaultCtr.end(); + + if (outputPropertyName) { + (result.data as any)[outputPropertyName] = discriminator; + } + + return result.data; + }), + ) as any; +} diff --git a/sdks/typescript/src/generated/types/smart-union.ts b/sdks/typescript/src/generated/types/smart-union.ts index a9db55cf..4809527f 100644 --- a/sdks/typescript/src/generated/types/smart-union.ts +++ b/sdks/typescript/src/generated/types/smart-union.ts @@ -5,6 +5,7 @@ // Not needed if lax mode import * as z from "zod/v4-mini"; import { startCountingDefaultToZeroValue } from "./default-to-zero-value.js"; +import { isUnknown as isDiscriminatedUnionUnknown } from "./discriminated-union.js"; import { RFCDate } from "./rfcdate.js"; import { startCountingUnrecognized } from "./unrecognized.js"; @@ -118,7 +119,7 @@ function countFieldsRecursive(parsed: unknown): number { while (index < queue.length) { const value = queue[index++]; - if (value === undefined) { + if (value === undefined || isDiscriminatedUnionUnknown(value)) { continue; } diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 4f83df96..4c60385e 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -1,7 +1,7 @@ from collections.abc import Iterator from agent_control_engine import list_evaluators -from agent_control_models import ConditionNode, ControlDefinition +from agent_control_models import ConditionNode, ControlDefinition, TemplateControlInput from agent_control_models.errors import ErrorCode, ValidationErrorItem from agent_control_models.server import ( AgentRef, @@ -15,6 +15,8 @@ PaginationInfo, PatchControlRequest, PatchControlResponse, + RenderControlTemplateRequest, + RenderControlTemplateResponse, SetControlDataRequest, SetControlDataResponse, ValidateControlDataRequest, @@ -38,6 +40,7 @@ from ..logging_utils import get_logger from ..models import Agent, AgentData, Control, agent_controls, agent_policies, policy_controls from ..services.control_definitions import parse_control_definition_or_api_error +from ..services.control_templates import remap_template_api_error, render_template_control_input from ..services.evaluator_utils import ( parse_evaluator_ref_full, validate_config_against_schema, @@ -53,6 +56,7 @@ _SCHEMA_VALIDATION_FAILED_MESSAGE = "Config does not satisfy the evaluator schema." router = APIRouter(prefix="/controls", tags=["controls"]) +template_router = APIRouter(prefix="/control-templates", tags=["controls"]) _logger = get_logger(__name__) @@ -96,6 +100,30 @@ def _serialize_control_definition(control_def: ControlDefinition) -> dict[str, o return data_json +def _is_template_backed_payload(data: object) -> bool: + """Return whether stored control JSON contains template metadata.""" + return isinstance(data, dict) and data.get("template") is not None + + +async def _render_and_validate_template_input( + template_input: TemplateControlInput, + *, + db: AsyncSession, + enabled: bool = True, +) -> ControlDefinition: + """Render a template-backed input and validate evaluator config.""" + rendered = render_template_control_input(template_input, enabled=enabled) + try: + await _validate_control_definition(rendered.control, db) + except APIValidationError as exc: + raise remap_template_api_error( + exc, + reverse_path_map=rendered.reverse_path_map, + template=template_input.template, + ) from exc + return rendered.control + + async def _validate_control_definition( control_def: ControlDefinition, db: AsyncSession ) -> None: @@ -256,6 +284,29 @@ async def _validate_control_definition( ) +@template_router.post( + "/render", + response_model=RenderControlTemplateResponse, + response_model_exclude_none=True, + summary="Render a control template preview", + response_description="Rendered control preview", +) +async def render_control_template( + request: RenderControlTemplateRequest, + db: AsyncSession = Depends(get_async_db), +) -> RenderControlTemplateResponse: + """Render a template-backed control without persisting it.""" + control_def = await _render_and_validate_template_input( + TemplateControlInput( + template=request.template, + template_values=request.template_values, + ), + db=db, + enabled=True, + ) + return RenderControlTemplateResponse(control=control_def) + + @router.put( "", dependencies=[Depends(require_admin_key)], @@ -294,8 +345,16 @@ async def create_control( hint="Choose a different name or update the existing control.", ) - await _validate_control_definition(request.data, db) - control_data = _serialize_control_definition(request.data) + if isinstance(request.data, TemplateControlInput): + control_def = await _render_and_validate_template_input( + request.data, + db=db, + enabled=True, + ) + else: + control_def = request.data + await _validate_control_definition(control_def, db) + control_data = _serialize_control_definition(control_def) control = Control(name=request.name, data=control_data) db.add(control) @@ -466,10 +525,43 @@ async def set_control_data( hint="Verify the control ID is correct and the control has been created.", ) - # Validate evaluator config using shared logic - await _validate_control_definition(request.data, db) + current_template_backed = _is_template_backed_payload(control.data) + if isinstance(request.data, TemplateControlInput): + current_enabled = True + if control.data: + try: + current_enabled = ControlDefinition.model_validate(control.data).enabled + except ValidationError: + current_enabled = True + control_def = await _render_and_validate_template_input( + request.data, + db=db, + enabled=current_enabled, + ) + else: + if current_template_backed: + raise ConflictError( + error_code=ErrorCode.CONTROL_IN_USE, + detail="Template-backed controls cannot be updated with raw control data in v1", + resource="Control", + resource_id=str(control_id), + hint=( + "Submit template input to update this control, or delete and recreate " + "it as a raw control." + ), + errors=[ + ValidationErrorItem( + resource="Control", + field="data", + code="template_backed_control_conflict", + message="Template-backed controls must be updated with template input.", + ) + ], + ) + control_def = request.data + await _validate_control_definition(control_def, db) - control.data = _serialize_control_definition(request.data) + control.data = _serialize_control_definition(control_def) try: await db.commit() except Exception: @@ -505,7 +597,14 @@ async def validate_control_data( Returns: ValidateControlDataResponse with success=True if valid """ - await _validate_control_definition(request.data, db) + if isinstance(request.data, TemplateControlInput): + await _render_and_validate_template_input( + request.data, + db=db, + enabled=True, + ) + else: + await _validate_control_definition(request.data, db) return ValidateControlDataResponse(success=True) @@ -520,6 +619,10 @@ async def list_controls( limit: int = Query(_DEFAULT_PAGINATION_LIMIT, ge=1, le=_MAX_PAGINATION_LIMIT), name: str | None = Query(None, description="Filter by name (partial, case-insensitive)"), enabled: bool | None = Query(None, description="Filter by enabled status"), + template_backed: bool | None = Query( + None, + description="Filter by whether the control is template-backed", + ), step_type: str | None = Query( None, description="Filter by step type (built-ins: 'tool', 'llm')" ), @@ -538,6 +641,7 @@ async def list_controls( limit: Maximum number of controls to return (default 20, max 100) name: Optional filter by name (partial, case-insensitive match) enabled: Optional filter by enabled status + template_backed: Optional filter by whether the control is template-backed step_type: Optional filter by step type (built-ins: 'tool', 'llm') stage: Optional filter by stage ('pre' or 'post') execution: Optional filter by execution ('server' or 'sdk') @@ -575,6 +679,12 @@ async def list_controls( # enabled=False: only include if explicitly false query = query.where(Control.data["enabled"].astext == "false") + if template_backed is not None: + if template_backed: + query = query.where(Control.data.has_key("template")) + else: + query = query.where(~Control.data.has_key("template")) + if step_type is not None: query = query.where( or_( @@ -618,6 +728,11 @@ async def list_controls( ) else: total_query = total_query.where(Control.data["enabled"].astext == "false") + if template_backed is not None: + if template_backed: + total_query = total_query.where(Control.data.has_key("template")) + else: + total_query = total_query.where(~Control.data.has_key("template")) if step_type is not None: total_query = total_query.where( or_( @@ -702,6 +817,7 @@ async def list_controls( step_types=scope.get("step_types"), stages=scope.get("stages"), tags=data.get("tags", []), + template_backed="template" in data, used_by_agent=control_agent_map.get(ctrl.id), used_by_agents_count=len(control_agent_names_map.get(ctrl.id, set())), ) diff --git a/server/src/agent_control_server/main.py b/server/src/agent_control_server/main.py index 09fd4ae6..cd80b6b6 100644 --- a/server/src/agent_control_server/main.py +++ b/server/src/agent_control_server/main.py @@ -20,6 +20,7 @@ from .db import AsyncSessionLocal from .endpoints.agents import router as agent_router from .endpoints.controls import router as control_router +from .endpoints.controls import template_router as control_template_router from .endpoints.evaluation import router as evaluation_router from .endpoints.evaluators import router as evaluator_router from .endpoints.observability import router as observability_router @@ -206,6 +207,11 @@ async def attach_version_header(request, call_next): # type: ignore[no-untyped- prefix=api_v1_prefix, dependencies=[Depends(require_api_key)], ) +app.include_router( + control_template_router, + prefix=api_v1_prefix, + dependencies=[Depends(require_api_key)], +) app.include_router( evaluation_router, prefix=api_v1_prefix, diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py new file mode 100644 index 00000000..4c7fb5d8 --- /dev/null +++ b/server/src/agent_control_server/services/control_templates.py @@ -0,0 +1,603 @@ +"""Template rendering and error-mapping helpers for control templates.""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from typing import cast + +import re2 +from agent_control_models import ( + ConditionNode, + ControlDefinition, + EnumTemplateParameter, + JsonValue, + TemplateControlInput, + TemplateDefinition, + TemplateParameterDefinition, + TemplateValue, +) +from agent_control_models.errors import ErrorCode, ValidationErrorItem +from pydantic import ValidationError + +from ..errors import APIValidationError +from .validation_paths import format_field_path + +_TEMPLATE_VALUE_MISSING = object() + + +@dataclass(frozen=True) +class RenderedTemplateControl: + """Rendered template result plus reverse mapping for validation errors.""" + + control: ControlDefinition + reverse_path_map: dict[str, str] + + +def _parameter_error( + parameter_name: str, + parameter_definition: TemplateParameterDefinition, + message: str, + *, + code: str = "template_parameter_invalid", + value: TemplateValue | None = None, +) -> APIValidationError: + """Create a parameter-focused validation error.""" + return APIValidationError( + error_code=ErrorCode.TEMPLATE_PARAMETER_INVALID, + detail=f"Invalid value for parameter '{parameter_definition.label}'", + resource="Control", + hint="Update the template parameter values and try again.", + errors=[ + ValidationErrorItem( + resource="Control", + field=f"template_values.{parameter_name}", + code=code, + message=message, + value=value, + parameter=parameter_name, + parameter_label=parameter_definition.label, + ) + ], + ) + + +def _render_error( + detail: str, + *, + field: str | None = None, + code: str = "template_render_error", + message: str | None = None, +) -> APIValidationError: + """Create a structural template rendering error.""" + errors = None + if message is not None: + errors = [ + ValidationErrorItem( + resource="Control", + field=field, + code=code, + message=message, + rendered_field=field, + ) + ] + + return APIValidationError( + error_code=ErrorCode.TEMPLATE_RENDER_ERROR, + detail=detail, + resource="Control", + hint="Update the template definition and try again.", + errors=errors, + ) + + +def _parameter_default( + parameter_definition: TemplateParameterDefinition, +) -> TemplateValue | object: + """Return the explicit default for a parameter, if any.""" + default = getattr(parameter_definition, "default", None) + return _TEMPLATE_VALUE_MISSING if default is None else cast(TemplateValue, default) + + +def _coerce_parameter_value( + parameter_name: str, + parameter_definition: TemplateParameterDefinition, + value: TemplateValue, +) -> TemplateValue: + """Validate a concrete parameter value against its parameter definition.""" + parameter_type = parameter_definition.type + if parameter_type == "string": + if not isinstance(value, str): + raise _parameter_error( + parameter_name, + parameter_definition, + f"Parameter '{parameter_definition.label}' must be a string.", + code="invalid_type", + value=value, + ) + return value + + if parameter_type == "string_list": + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise _parameter_error( + parameter_name, + parameter_definition, + f"Parameter '{parameter_definition.label}' must be a list of strings.", + code="invalid_type", + value=value, + ) + return list(value) + + if parameter_type == "enum": + if not isinstance(value, str): + raise _parameter_error( + parameter_name, + parameter_definition, + f"Parameter '{parameter_definition.label}' must be a string enum value.", + code="invalid_type", + value=value, + ) + enum_definition = cast(EnumTemplateParameter, parameter_definition) + if value not in enum_definition.allowed_values: + raise _parameter_error( + parameter_name, + parameter_definition, + ( + f"Parameter '{parameter_definition.label}' must be one of " + f"{enum_definition.allowed_values}." + ), + code="invalid_enum_value", + value=value, + ) + return value + + if parameter_type == "boolean": + if type(value) is not bool: + raise _parameter_error( + parameter_name, + parameter_definition, + f"Parameter '{parameter_definition.label}' must be a boolean.", + code="invalid_type", + value=value, + ) + return value + + if parameter_type == "regex_re2": + if not isinstance(value, str): + raise _parameter_error( + parameter_name, + parameter_definition, + f"Parameter '{parameter_definition.label}' must be a string regex pattern.", + code="invalid_type", + value=value, + ) + try: + re2.compile(value) + except re2.error as exc: + raise _parameter_error( + parameter_name, + parameter_definition, + ( + f"Invalid value for parameter '{parameter_definition.label}': " + f"Invalid regex pattern: {exc}" + ), + code="invalid_regex", + value=value, + ) from exc + return value + + raise _render_error( + detail=f"Unsupported template parameter type '{parameter_type}'", + code="unsupported_parameter_type", + message=f"Unsupported template parameter type '{parameter_type}'.", + ) + + +def _resolve_template_values( + template: TemplateDefinition, + template_values: Mapping[str, TemplateValue], +) -> dict[str, TemplateValue | object]: + """Resolve provided parameter values with defaults and validation.""" + unknown_keys = sorted(set(template_values) - set(template.parameters)) + if unknown_keys: + raise APIValidationError( + error_code=ErrorCode.TEMPLATE_PARAMETER_INVALID, + detail="Unknown template parameter(s) supplied", + resource="Control", + hint="Remove unknown template parameter keys and try again.", + errors=[ + ValidationErrorItem( + resource="Control", + field=f"template_values.{name}", + code="unknown_parameter", + message=f"Unknown template parameter '{name}'.", + parameter=name, + ) + for name in unknown_keys + ], + ) + + resolved: dict[str, TemplateValue | object] = {} + for parameter_name, parameter_definition in template.parameters.items(): + if parameter_name in template_values: + resolved[parameter_name] = _coerce_parameter_value( + parameter_name, + parameter_definition, + template_values[parameter_name], + ) + continue + + default_value = _parameter_default(parameter_definition) + if default_value is not _TEMPLATE_VALUE_MISSING: + resolved[parameter_name] = _coerce_parameter_value( + parameter_name, + parameter_definition, + cast(TemplateValue, default_value), + ) + continue + + if parameter_definition.required: + raise _parameter_error( + parameter_name, + parameter_definition, + f"Missing required parameter '{parameter_definition.label}'.", + code="missing_parameter", + ) + + resolved[parameter_name] = _TEMPLATE_VALUE_MISSING + + return resolved + + +def _format_rendered_path(path_parts: list[str | int]) -> str | None: + """Convert recursive path parts into dotted/bracketed field paths.""" + if not path_parts: + return None + + components: list[str] = [] + for part in path_parts: + if isinstance(part, int): + if not components: + components.append(f"[{part}]") + else: + components[-1] = f"{components[-1]}[{part}]" + continue + components.append(part) + return ".".join(components) + + +def _render_json_value( + value: JsonValue, + *, + resolved_values: Mapping[str, TemplateValue | object], + path_parts: list[str | int], + reverse_path_map: dict[str, str], + referenced_parameters: set[str], + template: TemplateDefinition, +) -> JsonValue: + """Render a JSON value by resolving $param binding objects recursively.""" + if isinstance(value, dict): + if "$param" in value: + if len(value) != 1 or not isinstance(value["$param"], str): + raise _render_error( + detail="Invalid $param binding object in template definition", + field=_format_rendered_path(path_parts), + code="invalid_param_binding", + message="A $param binding must be exactly {'$param': ''}.", + ) + + parameter_name = value["$param"] + if parameter_name not in template.parameters: + raise _render_error( + detail=f"Template references undefined parameter '{parameter_name}'", + field=_format_rendered_path(path_parts), + code="undefined_parameter_reference", + message=f"Template references undefined parameter '{parameter_name}'.", + ) + + resolved_value = resolved_values[parameter_name] + if resolved_value is _TEMPLATE_VALUE_MISSING: + parameter_definition = template.parameters[parameter_name] + raise _parameter_error( + parameter_name, + parameter_definition, + f"Missing value for parameter '{parameter_definition.label}'.", + code="missing_parameter", + ) + + referenced_parameters.add(parameter_name) + rendered_field = _format_rendered_path(path_parts) + if rendered_field is not None: + reverse_path_map[rendered_field] = parameter_name + return cast(JsonValue, resolved_value) + + return { + key: _render_json_value( + nested_value, + resolved_values=resolved_values, + path_parts=[*path_parts, key], + reverse_path_map=reverse_path_map, + referenced_parameters=referenced_parameters, + template=template, + ) + for key, nested_value in value.items() + } + + if isinstance(value, list): + return [ + _render_json_value( + nested_value, + resolved_values=resolved_values, + path_parts=[*path_parts, index], + reverse_path_map=reverse_path_map, + referenced_parameters=referenced_parameters, + template=template, + ) + for index, nested_value in enumerate(value) + ] + + return value + + +def _strip_request_prefix(field: str | None) -> str | None: + """Remove the request-level 'data.' prefix from raw control validation paths.""" + if field is None: + return None + if field.startswith("data."): + return field.removeprefix("data.") + if field == "data": + return None + return field + + +def _map_rendered_error_item( + item: ValidationErrorItem, + *, + reverse_path_map: Mapping[str, str], + template: TemplateDefinition, +) -> tuple[ValidationErrorItem, bool]: + """Map a rendered control validation item back to a template parameter when possible.""" + rendered_field = _strip_request_prefix(item.field) + if rendered_field is not None: + parameter_name = reverse_path_map.get(rendered_field) + if parameter_name is not None: + parameter_definition = template.parameters[parameter_name] + return ( + ValidationErrorItem( + resource=item.resource, + field=f"template_values.{parameter_name}", + code="template_parameter_invalid", + message=( + f"Invalid value for parameter '{parameter_definition.label}': " + f"{item.message}" + ), + value=item.value, + parameter=parameter_name, + parameter_label=parameter_definition.label, + rendered_field=rendered_field, + ), + True, + ) + + return ( + item.model_copy( + update={ + "field": rendered_field, + "rendered_field": rendered_field, + } + ), + False, + ) + + +def remap_template_api_error( + error: APIValidationError, + *, + reverse_path_map: Mapping[str, str], + template: TemplateDefinition, +) -> APIValidationError: + """Remap rendered-control validation errors to template parameter paths.""" + remapped_items: list[ValidationErrorItem] = [] + mapped_any = False + for item in error.errors or []: + remapped_item, mapped = _map_rendered_error_item( + item, + reverse_path_map=reverse_path_map, + template=template, + ) + remapped_items.append(remapped_item) + mapped_any = mapped_any or mapped + + return APIValidationError( + error_code=( + ErrorCode.TEMPLATE_PARAMETER_INVALID if mapped_any else ErrorCode.TEMPLATE_RENDER_ERROR + ), + detail=error.detail, + resource=error.resource, + hint=error.hint, + errors=remapped_items or error.errors, + ) + + +def _iter_condition_leaves( + node: ConditionNode, + *, + path: str = "condition", +) -> Iterator[tuple[str, ConditionNode]]: + """Yield each leaf condition with a dotted/bracketed field path.""" + if node.is_leaf(): + yield path, node + return + + if node.and_ is not None: + for index, child in enumerate(node.and_): + yield from _iter_condition_leaves(child, path=f"{path}.and[{index}]") + return + + if node.or_ is not None: + for index, child in enumerate(node.or_): + yield from _iter_condition_leaves(child, path=f"{path}.or[{index}]") + return + + if node.not_ is not None: + yield from _iter_condition_leaves(node.not_, path=f"{path}.not") + + +def _reject_agent_scoped_evaluators( + control: ControlDefinition, + *, + reverse_path_map: Mapping[str, str], + template: TemplateDefinition, +) -> None: + """Reject agent-scoped evaluator references in v1 templates.""" + for field_prefix, leaf in _iter_condition_leaves(control.condition): + leaf_parts = leaf.leaf_parts() + if leaf_parts is None: + continue + _, evaluator_spec = leaf_parts + if ":" not in evaluator_spec.name: + continue + + item = ValidationErrorItem( + resource="Control", + field=f"{field_prefix}.evaluator.name", + code="agent_scoped_evaluator_not_supported", + message=( + "Agent-scoped evaluators are not supported in control templates." + ), + ) + remapped_error, mapped = _map_rendered_error_item( + item, + reverse_path_map=reverse_path_map, + template=template, + ) + raise APIValidationError( + error_code=( + ErrorCode.TEMPLATE_PARAMETER_INVALID if mapped else ErrorCode.TEMPLATE_RENDER_ERROR + ), + detail="Agent-scoped evaluators are not supported in control templates", + resource="Control", + hint="Use a built-in or package-scoped evaluator in template-backed controls.", + errors=[remapped_error], + ) + + +def render_template_control_input( + template_input: TemplateControlInput, + *, + enabled: bool = True, +) -> RenderedTemplateControl: + """Render a template-backed control input into a concrete control definition.""" + template = template_input.template + definition_template = template.definition_template + if isinstance(definition_template, dict): + for forbidden_key in ("enabled", "name"): + if forbidden_key in definition_template: + raise _render_error( + detail=f"Templates must not define top-level '{forbidden_key}'", + field=forbidden_key, + code="forbidden_template_field", + message=( + f"Templates must not define top-level '{forbidden_key}'. " + "Manage it outside the template." + ), + ) + if "condition" not in definition_template and ( + "selector" in definition_template or "evaluator" in definition_template + ): + raise _render_error( + detail="Templates must use the canonical 'condition' format", + field="condition", + code="legacy_condition_format_not_supported", + message=( + "Templates must use the canonical 'condition' wrapper instead of " + "top-level selector/evaluator fields." + ), + ) + + resolved_values = _resolve_template_values(template, template_input.template_values) + reverse_path_map: dict[str, str] = {} + referenced_parameters: set[str] = set() + rendered_payload = _render_json_value( + definition_template, + resolved_values=resolved_values, + path_parts=[], + reverse_path_map=reverse_path_map, + referenced_parameters=referenced_parameters, + template=template, + ) + + unused_parameters = sorted(set(template.parameters) - referenced_parameters) + if unused_parameters: + raise APIValidationError( + error_code=ErrorCode.TEMPLATE_RENDER_ERROR, + detail="Template defines parameters that are never referenced", + resource="Control", + hint="Remove unused parameters or reference them in definition_template.", + errors=[ + ValidationErrorItem( + resource="Control", + field=f"template.parameters.{parameter_name}", + code="unused_template_parameter", + message=f"Template parameter '{parameter_name}' is never referenced.", + parameter=parameter_name, + parameter_label=template.parameters[parameter_name].label, + ) + for parameter_name in unused_parameters + ], + ) + + try: + rendered_control = ControlDefinition.model_validate(rendered_payload) + except ValidationError as exc: + mapped_items: list[ValidationErrorItem] = [] + mapped_any = False + for err in exc.errors(): + rendered_field = format_field_path(err.get("loc", ())) + remapped_item, mapped = _map_rendered_error_item( + ValidationErrorItem( + resource="Control", + field=rendered_field, + code=err.get("type", "validation_error"), + message=err.get("msg", "Validation failed"), + ), + reverse_path_map=reverse_path_map, + template=template, + ) + mapped_items.append(remapped_item) + mapped_any = mapped_any or mapped + + raise APIValidationError( + error_code=( + ErrorCode.TEMPLATE_PARAMETER_INVALID + if mapped_any + else ErrorCode.TEMPLATE_RENDER_ERROR + ), + detail="Rendered template did not produce a valid control definition", + resource="Control", + hint="Update the template structure or template parameter values and try again.", + errors=mapped_items, + ) from exc + + _reject_agent_scoped_evaluators( + rendered_control, + reverse_path_map=reverse_path_map, + template=template, + ) + + concrete_values = { + name: cast(TemplateValue, value) + for name, value in resolved_values.items() + if value is not _TEMPLATE_VALUE_MISSING + } + rendered_control = rendered_control.model_copy( + update={ + "enabled": enabled, + "template": template, + "template_values": concrete_values, + } + ) + return RenderedTemplateControl( + control=rendered_control, + reverse_path_map=reverse_path_map, + ) diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py new file mode 100644 index 00000000..cca5e621 --- /dev/null +++ b/server/tests/test_control_templates.py @@ -0,0 +1,169 @@ +"""Tests for template-backed control API flows.""" + +from __future__ import annotations + +import uuid +from copy import deepcopy + +from fastapi.testclient import TestClient + + +def _template_payload() -> dict[str, object]: + return { + "template": { + "description": "Regex denial template", + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + }, + "step_name": { + "type": "string", + "label": "Step Name", + "required": False, + "default": "templated-step", + }, + }, + "definition_template": { + "description": "Template-backed control", + "execution": "server", + "scope": { + "step_names": [{"$param": "step_name"}], + "stages": ["pre"], + }, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + "tags": ["template"], + }, + }, + "template_values": {"pattern": "hello"}, + } + + +def _create_template_control(client: TestClient) -> int: + response = client.put( + "/api/v1/controls", + json={ + "name": f"template-control-{uuid.uuid4()}", + "data": _template_payload(), + }, + ) + assert response.status_code == 200, response.text + return response.json()["control_id"] + + +def test_render_control_template_preview_returns_rendered_control(client: TestClient) -> None: + response = client.post("/api/v1/control-templates/render", json=_template_payload()) + + assert response.status_code == 200, response.text + control = response.json()["control"] + assert control["enabled"] is True + assert control["template"]["description"] == "Regex denial template" + assert control["template_values"] == { + "pattern": "hello", + "step_name": "templated-step", + } + assert control["condition"]["evaluator"]["config"]["pattern"] == "hello" + assert control["scope"]["step_names"] == ["templated-step"] + + +def test_create_template_backed_control_persists_template_metadata(client: TestClient) -> None: + control_id = _create_template_control(client) + + response = client.get(f"/api/v1/controls/{control_id}/data") + assert response.status_code == 200, response.text + data = response.json()["data"] + assert data["template"]["description"] == "Regex denial template" + assert data["template_values"]["pattern"] == "hello" + assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" + + +def test_template_update_preserves_enabled_value(client: TestClient) -> None: + control_id = _create_template_control(client) + + patch_response = client.patch( + f"/api/v1/controls/{control_id}", + json={"enabled": False}, + ) + assert patch_response.status_code == 200, patch_response.text + + updated_payload = _template_payload() + updated_payload["template_values"] = { + "pattern": "updated", + "step_name": "updated-step", + } + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": updated_payload}, + ) + assert put_response.status_code == 200, put_response.text + + get_response = client.get(f"/api/v1/controls/{control_id}/data") + assert get_response.status_code == 200, get_response.text + data = get_response.json()["data"] + assert data["enabled"] is False + assert data["template_values"] == { + "pattern": "updated", + "step_name": "updated-step", + } + assert data["condition"]["evaluator"]["config"]["pattern"] == "updated" + assert data["scope"]["step_names"] == ["updated-step"] + + +def test_template_validate_maps_missing_parameter_error(client: TestClient) -> None: + payload = _template_payload() + payload["template_values"] = {} + + response = client.post("/api/v1/controls/validate", json={"data": payload}) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" + assert any( + err.get("field") == "template_values.pattern" + and err.get("parameter") == "pattern" + for err in body.get("errors", []) + ) + + +def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> None: + control_id = _create_template_control(client) + raw_payload = deepcopy( + { + "description": "Raw replacement", + "enabled": True, + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "raw"}, + }, + }, + "action": {"decision": "deny"}, + } + ) + + response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": raw_payload}, + ) + + assert response.status_code == 409 + + +def test_list_controls_includes_template_backed_flag_and_filter(client: TestClient) -> None: + control_id = _create_template_control(client) + + response = client.get("/api/v1/controls", params={"template_backed": True}) + assert response.status_code == 200, response.text + controls = response.json()["controls"] + template_control = next(control for control in controls if control["id"] == control_id) + assert template_control["template_backed"] is True From 1dfb7f50d44f97a13eff39ecd3b883f3b174a2e9 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 21:47:18 -0700 Subject: [PATCH 02/25] fix: wire runtime parsing and template round-tripping --- engine/src/agent_control_engine/core.py | 3 +- models/src/agent_control_models/controls.py | 89 ++++++++++++++--- models/tests/test_control_templates.py | 55 ++++++++++- sdks/python/src/agent_control/__init__.py | 8 ++ sdks/python/src/agent_control/controls.py | 16 ++++ sdks/python/src/agent_control/evaluation.py | 9 +- sdks/python/tests/test_controls_api.py | 41 ++++++++ .../agent_control_server/endpoints/agents.py | 16 +--- .../endpoints/evaluation.py | 18 ++-- .../services/control_definitions.py | 28 +++++- .../agent_control_server/services/controls.py | 96 ++++++++++++++----- 11 files changed, 319 insertions(+), 60 deletions(-) diff --git a/engine/src/agent_control_engine/core.py b/engine/src/agent_control_engine/core.py index 7711430a..2600b7d7 100644 --- a/engine/src/agent_control_engine/core.py +++ b/engine/src/agent_control_engine/core.py @@ -16,6 +16,7 @@ from agent_control_models import ( ConditionNode, ControlDefinition, + ControlDefinitionRuntime, ControlMatch, EvaluationRequest, EvaluationResponse, @@ -47,7 +48,7 @@ class ControlWithIdentity(Protocol): id: int name: str - control: ControlDefinition + control: ControlDefinition | ControlDefinitionRuntime @dataclass diff --git a/models/src/agent_control_models/controls.py b/models/src/agent_control_models/controls.py index f56ad6c7..ca6b0a90 100644 --- a/models/src/agent_control_models/controls.py +++ b/models/src/agent_control_models/controls.py @@ -448,6 +448,24 @@ class ControlAction(BaseModel): MAX_CONDITION_DEPTH = 6 +def _validate_common_control_constraints( + condition: ConditionNode, + action: ControlAction, +) -> None: + """Validate control constraints shared by authoring and runtime models.""" + if condition.max_depth() > MAX_CONDITION_DEPTH: + raise ValueError( + f"Condition nesting depth exceeds maximum of {MAX_CONDITION_DEPTH}" + ) + + if ( + action.decision == "steer" + and not condition.is_leaf() + and action.steering_context is None + ): + raise ValueError("Composite steer controls require action.steering_context") + + class ConditionNode(BaseModel): """Recursive boolean condition tree for control evaluation.""" @@ -689,19 +707,7 @@ def canonicalize_legacy_condition_shape(cls, data: Any) -> Any: @model_validator(mode="after") def validate_condition_constraints(self) -> Self: """Validate cross-field control constraints.""" - if self.condition.max_depth() > MAX_CONDITION_DEPTH: - raise ValueError( - f"Condition nesting depth exceeds maximum of {MAX_CONDITION_DEPTH}" - ) - - if ( - self.action.decision == "steer" - and not self.condition.is_leaf() - and self.action.steering_context is None - ): - raise ValueError( - "Composite steer controls require action.steering_context" - ) + _validate_common_control_constraints(self.condition, self.action) has_template = self.template is not None has_template_values = self.template_values is not None if has_template != has_template_values: @@ -710,6 +716,15 @@ def validate_condition_constraints(self) -> Self: ) return self + def to_template_control_input(self) -> TemplateControlInput: + """Extract template-backed authoring input from a stored control definition.""" + if self.template is None or self.template_values is None: + raise ValueError("Control definition is not template-backed") + return TemplateControlInput( + template=self.template, + template_values=dict(self.template_values), + ) + def iter_condition_leaves(self) -> Iterator[ConditionNode]: """Yield leaf conditions in evaluation order.""" yield from self.condition.iter_leaves() @@ -808,6 +823,54 @@ class ControlDefinitionRuntime(BaseModel): action: ControlAction = Field(..., description="What action to take when control matches") tags: list[str] = Field(default_factory=list, description="Tags for categorization") + @model_validator(mode="before") + @classmethod + def canonicalize_legacy_condition_shape(cls, data: Any) -> Any: + """Accept legacy flat leaf payloads during runtime parsing.""" + return ControlDefinition.canonicalize_payload(data) + + @model_validator(mode="after") + def validate_condition_constraints(self) -> Self: + """Validate runtime-relevant control constraints.""" + _validate_common_control_constraints(self.condition, self.action) + return self + + def iter_condition_leaves(self) -> Iterator[ConditionNode]: + """Yield leaf conditions in evaluation order.""" + yield from self.condition.iter_leaves() + + def iter_condition_leaf_parts(self) -> Iterator[ConditionLeafParts]: + """Yield leaf selector/evaluator pairs in evaluation order.""" + yield from self.condition.iter_leaf_parts() + + def observability_identity(self) -> ControlObservabilityIdentity: + """Return a deterministic representative identity for observability.""" + all_evaluators: list[str] = [] + all_selector_paths: list[str] = [] + seen_evaluators: set[str] = set() + seen_selector_paths: set[str] = set() + leaf_count = 0 + + for selector, evaluator in self.iter_condition_leaf_parts(): + leaf_count += 1 + selector_path = selector.path or "*" + + if evaluator.name not in seen_evaluators: + seen_evaluators.add(evaluator.name) + all_evaluators.append(evaluator.name) + + if selector_path not in seen_selector_paths: + seen_selector_paths.add(selector_path) + all_selector_paths.append(selector_path) + + return ControlObservabilityIdentity( + selector_path=all_selector_paths[0] if all_selector_paths else None, + evaluator_name=all_evaluators[0] if all_evaluators else None, + leaf_count=leaf_count, + all_evaluators=all_evaluators, + all_selector_paths=all_selector_paths, + ) + def to_control_definition(self) -> ControlDefinition: """Promote a runtime control to the full public model when needed.""" return ControlDefinition.model_validate( diff --git a/models/tests/test_control_templates.py b/models/tests/test_control_templates.py index 24b85333..d2a7725c 100644 --- a/models/tests/test_control_templates.py +++ b/models/tests/test_control_templates.py @@ -3,7 +3,12 @@ from __future__ import annotations import pytest -from agent_control_models import ControlDefinition, TemplateControlInput, TemplateDefinition +from agent_control_models import ( + ControlDefinition, + ControlDefinitionRuntime, + TemplateControlInput, + TemplateDefinition, +) from agent_control_models.server import CreateControlRequest from pydantic import ValidationError @@ -97,3 +102,51 @@ def test_create_control_request_rejects_mixed_raw_and_template_payload() -> None }, } ) + + +def test_control_definition_can_round_trip_to_template_control_input() -> None: + control = ControlDefinition.model_validate( + { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "hello"}, + }, + }, + "action": {"decision": "deny"}, + "template": VALID_TEMPLATE, + "template_values": {"pattern": "hello"}, + } + ) + + template_input = control.to_template_control_input() + + assert template_input.template.parameters["pattern"].label == "Pattern" + assert template_input.template.parameters["pattern"].type == "regex_re2" + assert template_input.template.definition_template == VALID_TEMPLATE["definition_template"] + assert template_input.template_values == {"pattern": "hello"} + + +def test_control_definition_runtime_ignores_template_metadata() -> None: + runtime_control = ControlDefinitionRuntime.model_validate( + { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "hello"}, + }, + }, + "action": {"decision": "deny"}, + "template": VALID_TEMPLATE, + "template_values": {"pattern": "hello"}, + } + ) + + assert runtime_control.execution == "server" + assert runtime_control.action.decision == "deny" diff --git a/sdks/python/src/agent_control/__init__.py b/sdks/python/src/agent_control/__init__.py index 22c475e8..65e7c64e 100644 --- a/sdks/python/src/agent_control/__init__.py +++ b/sdks/python/src/agent_control/__init__.py @@ -164,6 +164,13 @@ def get_server_controls() -> list[dict[str, Any]] | None: return state.server_controls +def to_template_control_input( + data: dict[str, Any] | ControlDefinition, +) -> TemplateControlInput: + """Convert stored control data into template authoring input.""" + return controls.to_template_control_input(data) + + def _publish_server_controls( controls: list[dict[str, Any]] | None, ) -> list[dict[str, Any]] | None: @@ -1316,6 +1323,7 @@ async def main(): "update_control", "validate_control_data", "render_control_template", + "to_template_control_input", # Decorator "control", "ControlViolationError", diff --git a/sdks/python/src/agent_control/controls.py b/sdks/python/src/agent_control/controls.py index bff95b0a..a9a19d16 100644 --- a/sdks/python/src/agent_control/controls.py +++ b/sdks/python/src/agent_control/controls.py @@ -265,6 +265,22 @@ async def render_control_template( return cast(dict[str, Any], response.json()) +def to_template_control_input( + data: dict[str, Any] | ControlDefinition, +) -> TemplateControlInput: + """Convert stored control data into template authoring input. + + This is the supported reshape path for template-backed controls returned by + ``GET /controls/{id}`` or ``GET /controls/{id}/data`` before submitting them + back to ``set_control_data``. + """ + if isinstance(data, ControlDefinition): + control_def = data + else: + control_def = ControlDefinition.model_validate(data) + return control_def.to_template_control_input() + + async def add_rule_to_control( client: AgentControlClient, control_id: int, diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index d76af177..d8df7c79 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -8,6 +8,7 @@ from agent_control_engine.core import ControlEngine from agent_control_models import ( ControlDefinition, + ControlDefinitionRuntime, ControlExecutionEvent, ControlMatch, EvaluationRequest, @@ -33,7 +34,7 @@ def _observability_metadata( - control_def: ControlDefinition, + control_def: ControlDefinition | ControlDefinitionRuntime, ) -> tuple[str | None, str | None, dict[str, object]]: """Return representative event fields plus full composite context.""" identity = control_def.observability_identity() @@ -159,7 +160,7 @@ class _ControlAdapter: id: int name: str - control: "ControlDefinition" + control: "ControlDefinition | ControlDefinitionRuntime" def _get_applicable_controls( @@ -193,7 +194,7 @@ def _has_applicable_prefiltered_server_controls( for control in server_control_payloads: try: - control_def = ControlDefinition.model_validate(control["control"]) + control_def = ControlDefinitionRuntime.model_validate(control["control"]) parsed_server_controls.append( _ControlAdapter( id=control["id"], @@ -315,7 +316,7 @@ async def check_evaluation_with_local( continue try: - control_def = ControlDefinition.model_validate(control_data) + control_def = ControlDefinitionRuntime.model_validate(control_data) for _, evaluator_spec in control_def.iter_condition_leaf_parts(): evaluator_name = evaluator_spec.name diff --git a/sdks/python/tests/test_controls_api.py b/sdks/python/tests/test_controls_api.py index fbf15811..e19cdaed 100644 --- a/sdks/python/tests/test_controls_api.py +++ b/sdks/python/tests/test_controls_api.py @@ -107,3 +107,44 @@ async def test_render_control_template_calls_preview_endpoint() -> None: "template_values": {}, }, ) + + +def test_to_template_control_input_reshapes_stored_control_data() -> None: + template_input = agent_control.controls.to_template_control_input( + { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "hello"}, + }, + }, + "action": {"decision": "deny"}, + "template": { + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + } + }, + "definition_template": { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {"pattern": "hello"}, + } + ) + + assert isinstance(template_input, TemplateControlInput) + assert template_input.template_values == {"pattern": "hello"} diff --git a/server/src/agent_control_server/endpoints/agents.py b/server/src/agent_control_server/endpoints/agents.py index d88a9f0c..8fc63c32 100644 --- a/server/src/agent_control_server/endpoints/agents.py +++ b/server/src/agent_control_server/endpoints/agents.py @@ -1,11 +1,12 @@ from collections.abc import Sequence -from typing import Any, Protocol +from typing import Any from agent_control_engine import list_evaluators from agent_control_models.agent import Agent as APIAgent from agent_control_models.agent import StepSchema -from agent_control_models.controls import ControlDefinition +from agent_control_models.controls import ControlDefinitionRuntime from agent_control_models.errors import ErrorCode, ValidationErrorItem +from agent_control_models.policy import Control as APIControl from agent_control_models.server import ( AgentControlsResponse, AgentSummary, @@ -82,13 +83,6 @@ type StepKeyTuple = tuple[str, str] -class _ControlWithDefinition(Protocol): - """Minimal control shape needed for evaluator dependency scans.""" - - name: str - control: ControlDefinition - - # ============================================================================= # List Agents Models # ============================================================================= @@ -119,7 +113,7 @@ def _validate_controls_for_agent(agent: Agent, controls: list[Control]) -> list[ continue try: - control_definition = ControlDefinition.model_validate(control.data) + control_definition = ControlDefinitionRuntime.model_validate(control.data) except ValidationError: errors.append(f"Control '{control.name}' has corrupted data") continue @@ -165,7 +159,7 @@ def _validate_controls_for_agent(agent: Agent, controls: list[Control]) -> list[ def _find_referencing_controls_for_removed_evaluators( - controls: Sequence[_ControlWithDefinition], + controls: Sequence[APIControl], agent_name: str, remove_evaluator_set: set[str], ) -> list[tuple[str, str]]: diff --git a/server/src/agent_control_server/endpoints/evaluation.py b/server/src/agent_control_server/endpoints/evaluation.py index c92ea315..ddade004 100644 --- a/server/src/agent_control_server/endpoints/evaluation.py +++ b/server/src/agent_control_server/endpoints/evaluation.py @@ -7,6 +7,7 @@ from agent_control_engine.core import ControlEngine from agent_control_models import ( ControlDefinition, + ControlDefinitionRuntime, ControlExecutionEvent, ControlMatch, EvaluationRequest, @@ -23,7 +24,7 @@ from ..logging_utils import get_logger from ..models import Agent from ..observability.ingest.base import EventIngestor -from ..services.controls import list_controls_for_agent +from ..services.controls import list_runtime_controls_for_agent from .observability import get_event_ingestor router = APIRouter(prefix="/evaluation", tags=["evaluation"]) @@ -43,7 +44,12 @@ class ControlAdapter: """Adapts API Control to Engine ControlWithIdentity protocol.""" - def __init__(self, id: int, name: str, control: ControlDefinition): + def __init__( + self, + id: int, + name: str, + control: ControlDefinition | ControlDefinitionRuntime, + ): self.id = id self.name = name self.control = control @@ -201,18 +207,18 @@ async def evaluate( ) agent_name = agent.name - # Fetch controls for the agent (already validated as ControlDefinition) - api_controls = await list_controls_for_agent( + # Fetch controls for the agent using the runtime model for evaluation. + runtime_controls = await list_runtime_controls_for_agent( request.agent_name, db, allow_invalid_step_name_regex=True, ) # Build control lookup for observability - control_lookup = {c.id: c for c in api_controls} + control_lookup = {c.id: c for c in runtime_controls} # Adapt controls for the engine - engine_controls = [ControlAdapter(c.id, c.name, c.control) for c in api_controls] + engine_controls = [ControlAdapter(c.id, c.name, c.control) for c in runtime_controls] # Execute Control Engine (parallel with cancel-on-deny) engine = ControlEngine(engine_controls) diff --git a/server/src/agent_control_server/services/control_definitions.py b/server/src/agent_control_server/services/control_definitions.py index c73c8287..23d65202 100644 --- a/server/src/agent_control_server/services/control_definitions.py +++ b/server/src/agent_control_server/services/control_definitions.py @@ -5,7 +5,7 @@ from collections.abc import Mapping, Sequence from typing import Any, cast -from agent_control_models import ControlDefinition +from agent_control_models import ControlDefinition, ControlDefinitionRuntime from agent_control_models.errors import ErrorCode, ValidationErrorItem from pydantic import ValidationError @@ -62,3 +62,29 @@ def parse_control_definition_or_api_error( hint=hint, errors=build_control_validation_errors(exc, field_prefix=field_prefix), ) from exc + + +def parse_runtime_control_definition_or_api_error( + data: Any, + *, + detail: str, + hint: str, + resource_id: str | None = None, + context: Mapping[str, Any] | None = None, + field_prefix: str | None = "data", +) -> ControlDefinitionRuntime: + """Parse stored runtime control data or raise a structured CORRUPTED_DATA error.""" + try: + return ControlDefinitionRuntime.model_validate( + data, + context=dict(context) if context else None, + ) + except ValidationError as exc: + raise APIValidationError( + error_code=ErrorCode.CORRUPTED_DATA, + detail=detail, + resource="Control", + resource_id=resource_id, + hint=hint, + errors=build_control_validation_errors(exc, field_prefix=field_prefix), + ) from exc diff --git a/server/src/agent_control_server/services/controls.py b/server/src/agent_control_server/services/controls.py index 5ab97237..44e80020 100644 --- a/server/src/agent_control_server/services/controls.py +++ b/server/src/agent_control_server/services/controls.py @@ -1,40 +1,34 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import dataclass +from agent_control_models import ControlDefinitionRuntime from agent_control_models.policy import Control as APIControl from sqlalchemy import select, union from sqlalchemy.ext.asyncio import AsyncSession from ..models import Control, agent_controls, agent_policies, policy_controls -from .control_definitions import parse_control_definition_or_api_error +from .control_definitions import ( + parse_control_definition_or_api_error, + parse_runtime_control_definition_or_api_error, +) -async def list_controls_for_policy(policy_id: int, db: AsyncSession) -> list[Control]: - """Return DB Control objects for all controls directly associated with a policy.""" - stmt = ( - select(Control) - .join(policy_controls, Control.id == policy_controls.c.control_id) - .where(policy_controls.c.policy_id == policy_id) - ) - result = await db.execute(stmt) - return list(result.scalars().unique().all()) +@dataclass(frozen=True) +class RuntimeControl: + """Internal runtime control payload for evaluation hot paths.""" + id: int + name: str + control: ControlDefinitionRuntime -async def list_controls_for_agent( + +async def _list_db_controls_for_agent( agent_name: str, db: AsyncSession, - *, - allow_invalid_step_name_regex: bool = False, -) -> list[APIControl]: - """Return API Control models for controls associated with the agent. - - Active controls are the de-duplicated union of: - - controls inherited from all assigned policies - - controls directly associated with the agent - - Note: Invalid ControlDefinition data triggers an APIValidationError. - """ +) -> Sequence[Control]: + """Return DB Control rows for the active controls associated with an agent.""" policy_control_ids = ( select(policy_controls.c.control_id.label("control_id")) .select_from( @@ -56,7 +50,35 @@ async def list_controls_for_agent( ) result = await db.execute(stmt) - db_controls: Sequence[Control] = result.scalars().unique().all() + return result.scalars().unique().all() + + +async def list_controls_for_policy(policy_id: int, db: AsyncSession) -> list[Control]: + """Return DB Control objects for all controls directly associated with a policy.""" + stmt = ( + select(Control) + .join(policy_controls, Control.id == policy_controls.c.control_id) + .where(policy_controls.c.policy_id == policy_id) + ) + result = await db.execute(stmt) + return list(result.scalars().unique().all()) + + +async def list_controls_for_agent( + agent_name: str, + db: AsyncSession, + *, + allow_invalid_step_name_regex: bool = False, +) -> list[APIControl]: + """Return API Control models for controls associated with the agent. + + Active controls are the de-duplicated union of: + - controls inherited from all assigned policies + - controls directly associated with the agent + + Note: Invalid ControlDefinition data triggers an APIValidationError. + """ + db_controls = await _list_db_controls_for_agent(agent_name, db) # Map DB Control to API Control, raising on invalid definitions api_controls: list[APIControl] = [] @@ -76,3 +98,31 @@ async def list_controls_for_agent( ) api_controls.append(APIControl(id=c.id, name=c.name, control=control_def)) return api_controls + + +async def list_runtime_controls_for_agent( + agent_name: str, + db: AsyncSession, + *, + allow_invalid_step_name_regex: bool = False, +) -> list[RuntimeControl]: + """Return runtime-parsed controls for evaluation hot paths.""" + db_controls = await _list_db_controls_for_agent(agent_name, db) + + runtime_controls: list[RuntimeControl] = [] + for c in db_controls: + context = ( + {"allow_invalid_step_name_regex": True} + if allow_invalid_step_name_regex + else None + ) + control_def = parse_runtime_control_definition_or_api_error( + c.data, + detail=f"Control '{c.name}' has corrupted data", + resource_id=str(c.id), + hint=f"Update the control data using PUT /api/v1/controls/{c.id}/data.", + context=context, + field_prefix="data", + ) + runtime_controls.append(RuntimeControl(id=c.id, name=c.name, control=control_def)) + return runtime_controls From 19aa09635d176d73ae6743b6f10aa1bd3a6220a3 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 21:57:58 -0700 Subject: [PATCH 03/25] fix: tighten template control API contracts --- models/src/agent_control_models/errors.py | 2 + models/src/agent_control_models/server.py | 4 +- .../endpoints/controls.py | 2 +- .../services/control_templates.py | 2 + server/tests/test_control_templates.py | 174 ++++++++++++++++++ 5 files changed, 182 insertions(+), 2 deletions(-) diff --git a/models/src/agent_control_models/errors.py b/models/src/agent_control_models/errors.py index 8f37b08b..d51b0638 100644 --- a/models/src/agent_control_models/errors.py +++ b/models/src/agent_control_models/errors.py @@ -68,6 +68,7 @@ class ErrorCode(StrEnum): CONTROL_NAME_CONFLICT = "CONTROL_NAME_CONFLICT" EVALUATOR_NAME_CONFLICT = "EVALUATOR_NAME_CONFLICT" CONTROL_IN_USE = "CONTROL_IN_USE" + CONTROL_TEMPLATE_CONFLICT = "CONTROL_TEMPLATE_CONFLICT" EVALUATOR_IN_USE = "EVALUATOR_IN_USE" SCHEMA_INCOMPATIBLE = "SCHEMA_INCOMPATIBLE" @@ -370,6 +371,7 @@ def make_error_type(error_code: ErrorCode) -> str: ErrorCode.CONTROL_NAME_CONFLICT: "Control Name Already Exists", ErrorCode.EVALUATOR_NAME_CONFLICT: "Evaluator Name Conflict", ErrorCode.CONTROL_IN_USE: "Control In Use", + ErrorCode.CONTROL_TEMPLATE_CONFLICT: "Control Template Conflict", ErrorCode.EVALUATOR_IN_USE: "Evaluator In Use", ErrorCode.SCHEMA_INCOMPATIBLE: "Schema Incompatible", # Validation errors diff --git a/models/src/agent_control_models/server.py b/models/src/agent_control_models/server.py index c7386cad..a0e099de 100644 --- a/models/src/agent_control_models/server.py +++ b/models/src/agent_control_models/server.py @@ -1,7 +1,7 @@ from enum import StrEnum from typing import Annotated, Any -from pydantic import BeforeValidator, Field, StringConstraints, TypeAdapter +from pydantic import BeforeValidator, ConfigDict, Field, StringConstraints, TypeAdapter from .agent import Agent, StepSchema from .base import BaseModel @@ -346,6 +346,8 @@ class ValidateControlDataResponse(BaseModel): class RenderControlTemplateRequest(BaseModel): """Request to render a template-backed control without persisting it.""" + model_config = ConfigDict(extra="forbid") + template: TemplateDefinition = Field(..., description="Template definition to render") template_values: dict[str, TemplateValue] = Field( default_factory=dict, diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 4c60385e..a8a0cf3e 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -541,7 +541,7 @@ async def set_control_data( else: if current_template_backed: raise ConflictError( - error_code=ErrorCode.CONTROL_IN_USE, + error_code=ErrorCode.CONTROL_TEMPLATE_CONFLICT, detail="Template-backed controls cannot be updated with raw control data in v1", resource="Control", resource_id=str(control_id), diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index 4c7fb5d8..5dc4a1d6 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -95,6 +95,8 @@ def _parameter_default( parameter_definition: TemplateParameterDefinition, ) -> TemplateValue | object: """Return the explicit default for a parameter, if any.""" + # TemplateValue intentionally excludes None, so None continues to mean + # "no default provided" for current v1 parameter types. default = getattr(parameter_definition, "default", None) return _TEMPLATE_VALUE_MISSING if default is None else cast(TemplateValue, default) diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index cca5e621..da4a0e92 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -46,6 +46,23 @@ def _template_payload() -> dict[str, object]: } +def _raw_control_payload(pattern: str = "raw") -> dict[str, object]: + return { + "description": "Raw control", + "enabled": True, + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": pattern}, + }, + }, + "action": {"decision": "deny"}, + } + + def _create_template_control(client: TestClient) -> int: response = client.put( "/api/v1/controls", @@ -58,6 +75,18 @@ def _create_template_control(client: TestClient) -> int: return response.json()["control_id"] +def _create_raw_control(client: TestClient) -> int: + response = client.put( + "/api/v1/controls", + json={ + "name": f"raw-control-{uuid.uuid4()}", + "data": _raw_control_payload(), + }, + ) + assert response.status_code == 200, response.text + return response.json()["control_id"] + + def test_render_control_template_preview_returns_rendered_control(client: TestClient) -> None: response = client.post("/api/v1/control-templates/render", json=_template_payload()) @@ -84,6 +113,23 @@ def test_create_template_backed_control_persists_template_metadata(client: TestC assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" +def test_raw_control_can_be_replaced_with_template_backed_control(client: TestClient) -> None: + control_id = _create_raw_control(client) + + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": _template_payload()}, + ) + assert put_response.status_code == 200, put_response.text + + get_response = client.get(f"/api/v1/controls/{control_id}/data") + assert get_response.status_code == 200, get_response.text + data = get_response.json()["data"] + assert data["template"]["description"] == "Regex denial template" + assert data["template_values"]["pattern"] == "hello" + assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" + + def test_template_update_preserves_enabled_value(client: TestClient) -> None: control_id = _create_template_control(client) @@ -157,6 +203,7 @@ def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> N ) assert response.status_code == 409 + assert response.json()["error_code"] == "CONTROL_TEMPLATE_CONFLICT" def test_list_controls_includes_template_backed_flag_and_filter(client: TestClient) -> None: @@ -167,3 +214,130 @@ def test_list_controls_includes_template_backed_flag_and_filter(client: TestClie controls = response.json()["controls"] template_control = next(control for control in controls if control["id"] == control_id) assert template_control["template_backed"] is True + + +def test_list_controls_template_backed_false_returns_only_raw_controls(client: TestClient) -> None: + template_control_id = _create_template_control(client) + raw_control_id = _create_raw_control(client) + + response = client.get("/api/v1/controls", params={"template_backed": False}) + assert response.status_code == 200, response.text + control_ids = {control["id"] for control in response.json()["controls"]} + assert raw_control_id in control_ids + assert template_control_id not in control_ids + + +def test_render_control_template_rejects_extra_request_fields(client: TestClient) -> None: + payload = _template_payload() + payload["execution"] = "server" + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + assert response.json()["error_code"] == "VALIDATION_ERROR" + + +def test_render_control_template_maps_invalid_regex_parameter(client: TestClient) -> None: + payload = _template_payload() + payload["template_values"] = {"pattern": "["} + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" + assert any( + err.get("field") == "template_values.pattern" + and err.get("parameter") == "pattern" + for err in body.get("errors", []) + ) + + +def test_render_control_template_rejects_malformed_param_binding(client: TestClient) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["pattern"] = { # type: ignore[index] + "$param": "pattern", + "extra": True, + } + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any(err.get("code") == "invalid_param_binding" for err in body.get("errors", [])) + + +def test_render_control_template_rejects_non_string_param_reference(client: TestClient) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["pattern"] = { # type: ignore[index] + "$param": 123, + } + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any(err.get("code") == "invalid_param_binding" for err in body.get("errors", [])) + + +def test_render_control_template_rejects_agent_scoped_evaluator(client: TestClient) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"]["condition"]["evaluator"]["name"] = "agent-x:custom" # type: ignore[index] + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("code") == "agent_scoped_evaluator_not_supported" + for err in body.get("errors", []) + ) + + +def test_render_control_template_rejects_legacy_flat_format(client: TestClient) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"] = { # type: ignore[index] + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + "action": {"decision": "deny"}, + } + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("code") == "legacy_condition_format_not_supported" + for err in body.get("errors", []) + ) + + +def test_render_control_template_rejects_invalid_parameter_name_at_api_boundary( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["parameters"] = { # type: ignore[index] + "bad.name": { + "type": "string", + "label": "Bad Name", + } + } + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + assert response.json()["error_code"] == "VALIDATION_ERROR" From 3fcdd9210603a70ecff38ea0c344a687f4f85f41 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 22:14:55 -0700 Subject: [PATCH 04/25] test: add behavioral coverage for control templates --- sdks/python/tests/test_local_evaluation.py | 88 +++++++++ server/tests/test_control_templates.py | 204 ++++++++++++++++++++- 2 files changed, 289 insertions(+), 3 deletions(-) diff --git a/sdks/python/tests/test_local_evaluation.py b/sdks/python/tests/test_local_evaluation.py index 2a012bab..f124fef4 100644 --- a/sdks/python/tests/test_local_evaluation.py +++ b/sdks/python/tests/test_local_evaluation.py @@ -93,6 +93,33 @@ def make_control_dict( } +def add_template_metadata(control: dict[str, Any], *, pattern: str = "test") -> dict[str, Any]: + """Attach realistic template metadata to a cached control payload.""" + control["control"]["template"] = { + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + } + }, + "definition_template": { + "description": "Template-backed control", + "execution": control["control"]["execution"], + "scope": control["control"]["scope"], + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": control["control"]["action"], + }, + } + control["control"]["template_values"] = {"pattern": pattern} + return control + + NON_APPLICABLE_CONTROL_CASES = [ pytest.param({"enabled": False}, id="disabled"), pytest.param({"stage": "post"}, id="stage_mismatch"), @@ -271,6 +298,37 @@ async def test_server_only_controls_calls_server(self, agent_name, llm_payload): assert result.is_safe is True + @pytest.mark.asyncio + async def test_server_only_template_backed_controls_still_call_server( + self, + agent_name, + llm_payload, + ): + """Template metadata must not break routing for server-executed cached controls.""" + controls = [ + add_template_metadata( + make_control_dict(1, "templated_server_ctrl", execution="server"), + ), + ] + + client = MagicMock(spec=AgentControlClient) + mock_response = MagicMock() + mock_response.json.return_value = {"is_safe": True, "confidence": 1.0} + mock_response.raise_for_status = MagicMock() + client.http_client = AsyncMock() + client.http_client.post = AsyncMock(return_value=mock_response) + + result = await check_evaluation_with_local( + client=client, + agent_name=agent_name, + step=llm_payload, + stage="pre", + controls=controls, + ) + + client.http_client.post.assert_called_once() + assert result.is_safe is True + @pytest.mark.asyncio @pytest.mark.parametrize( "control_kwargs", @@ -447,6 +505,36 @@ async def test_local_legacy_flat_control_executes_locally(self, agent_name, llm_ assert len(result.matches) == 1 assert result.matches[0].control_name == "legacy_local_ctrl" + @pytest.mark.asyncio + async def test_template_backed_local_control_executes_locally( + self, + agent_name, + llm_payload, + ): + """Template metadata should be ignored by runtime parsing in local evaluation.""" + controls = [ + add_template_metadata( + make_control_dict(1, "templated_local_ctrl", execution="sdk", pattern=r"test"), + ), + ] + client = MagicMock(spec=AgentControlClient) + client.http_client = AsyncMock() + client.http_client.post = AsyncMock() + + result = await check_evaluation_with_local( + client=client, + agent_name=agent_name, + step=llm_payload, + stage="pre", + controls=controls, + ) + + client.http_client.post.assert_not_called() + assert result.is_safe is False + assert result.matches is not None + assert len(result.matches) == 1 + assert result.matches[0].control_name == "templated_local_ctrl" + @pytest.mark.asyncio async def test_local_deny_short_circuits(self, agent_name, llm_payload): """Local deny should return immediately without calling server.""" diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index da4a0e92..6c878239 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -5,6 +5,7 @@ import uuid from copy import deepcopy +from agent_control_models import EvaluationRequest, Step from fastapi.testclient import TestClient @@ -46,6 +47,49 @@ def _template_payload() -> dict[str, object]: } +def _defaults_only_template_payload() -> dict[str, object]: + return { + "template": { + "description": "List evaluator template", + "parameters": { + "values": { + "type": "string_list", + "label": "Values", + "default": ["secret", "blocked"], + }, + "logic": { + "type": "enum", + "label": "Logic", + "allowed_values": ["any", "all"], + "default": "any", + }, + "case_sensitive": { + "type": "boolean", + "label": "Case Sensitive", + "default": False, + }, + }, + "definition_template": { + "description": "Defaulted list control", + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "list", + "config": { + "values": {"$param": "values"}, + "logic": {"$param": "logic"}, + "case_sensitive": {"$param": "case_sensitive"}, + }, + }, + }, + "action": {"decision": "deny"}, + }, + } + } + + def _raw_control_payload(pattern: str = "raw") -> dict[str, object]: return { "description": "Raw control", @@ -64,15 +108,46 @@ def _raw_control_payload(pattern: str = "raw") -> dict[str, object]: def _create_template_control(client: TestClient) -> int: + control_id, _ = _create_template_control_with_name(client) + return control_id + + +def _create_template_control_with_name(client: TestClient) -> tuple[int, str]: + control_name = f"template-control-{uuid.uuid4()}" response = client.put( "/api/v1/controls", json={ - "name": f"template-control-{uuid.uuid4()}", + "name": control_name, "data": _template_payload(), }, ) assert response.status_code == 200, response.text - return response.json()["control_id"] + return response.json()["control_id"], control_name + + +def _assign_control_to_agent( + client: TestClient, + control_id: int, + *, + agent_name: str | None = None, +) -> str: + policy_response = client.put("/api/v1/policies", json={"name": f"policy-{uuid.uuid4()}"}) + assert policy_response.status_code == 200, policy_response.text + policy_id = policy_response.json()["policy_id"] + + add_control_response = client.post(f"/api/v1/policies/{policy_id}/controls/{control_id}") + assert add_control_response.status_code == 200, add_control_response.text + + effective_agent_name = agent_name or f"template-agent-{uuid.uuid4().hex[:12]}" + init_response = client.post( + "/api/v1/agents/initAgent", + json={"agent": {"agent_name": effective_agent_name}, "steps": []}, + ) + assert init_response.status_code == 200, init_response.text + + assign_response = client.post(f"/api/v1/agents/{effective_agent_name}/policy/{policy_id}") + assert assign_response.status_code == 200, assign_response.text + return effective_agent_name def _create_raw_control(client: TestClient) -> int: @@ -102,6 +177,26 @@ def test_render_control_template_preview_returns_rendered_control(client: TestCl assert control["scope"]["step_names"] == ["templated-step"] +def test_render_control_template_preview_uses_defaults_when_values_omitted( + client: TestClient, +) -> None: + response = client.post("/api/v1/control-templates/render", json=_defaults_only_template_payload()) + + assert response.status_code == 200, response.text + control = response.json()["control"] + assert control["template_values"] == { + "values": ["secret", "blocked"], + "logic": "any", + "case_sensitive": False, + } + assert control["condition"]["evaluator"]["name"] == "list" + assert control["condition"]["evaluator"]["config"] == { + "values": ["secret", "blocked"], + "logic": "any", + "case_sensitive": False, + } + + def test_create_template_backed_control_persists_template_metadata(client: TestClient) -> None: control_id = _create_template_control(client) @@ -109,10 +204,38 @@ def test_create_template_backed_control_persists_template_metadata(client: TestC assert response.status_code == 200, response.text data = response.json()["data"] assert data["template"]["description"] == "Regex denial template" - assert data["template_values"]["pattern"] == "hello" + assert data["template_values"] == { + "pattern": "hello", + "step_name": "templated-step", + } assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" +def test_template_backed_control_evaluates_after_policy_attachment(client: TestClient) -> None: + control_id, control_name = _create_template_control_with_name(client) + agent_name = _assign_control_to_agent(client, control_id) + + safe_request = EvaluationRequest( + agent_name=agent_name, + step=Step(type="llm", name="other-step", input="hello", output=None), + stage="pre", + ) + safe_response = client.post("/api/v1/evaluation", json=safe_request.model_dump(mode="json")) + assert safe_response.status_code == 200, safe_response.text + assert safe_response.json()["is_safe"] is True + + deny_request = EvaluationRequest( + agent_name=agent_name, + step=Step(type="llm", name="templated-step", input="hello", output=None), + stage="pre", + ) + deny_response = client.post("/api/v1/evaluation", json=deny_request.model_dump(mode="json")) + assert deny_response.status_code == 200, deny_response.text + body = deny_response.json() + assert body["is_safe"] is False + assert body["matches"][0]["control_name"] == control_name + + def test_raw_control_can_be_replaced_with_template_backed_control(client: TestClient) -> None: control_id = _create_raw_control(client) @@ -178,6 +301,22 @@ def test_template_validate_maps_missing_parameter_error(client: TestClient) -> N ) +def test_render_control_template_rejects_unknown_template_value_key(client: TestClient) -> None: + payload = _template_payload() + payload["template_values"] = {"pattern": "hello", "unknown": "value"} + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" + assert any( + err.get("field") == "template_values.unknown" + and err.get("code") == "unknown_parameter" + for err in body.get("errors", []) + ) + + def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> None: control_id = _create_template_control(client) raw_payload = deepcopy( @@ -284,6 +423,27 @@ def test_render_control_template_rejects_non_string_param_reference(client: Test assert any(err.get("code") == "invalid_param_binding" for err in body.get("errors", [])) +def test_render_control_template_rejects_unused_parameter(client: TestClient) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["parameters"]["unused"] = { # type: ignore[index] + "type": "string", + "label": "Unused", + "default": "still-unused", + } + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("field") == "template.parameters.unused" + and err.get("code") == "unused_template_parameter" + for err in body.get("errors", []) + ) + + def test_render_control_template_rejects_agent_scoped_evaluator(client: TestClient) -> None: payload = _template_payload() payload["template"] = deepcopy(payload["template"]) @@ -300,6 +460,25 @@ def test_render_control_template_rejects_agent_scoped_evaluator(client: TestClie ) +def test_render_control_template_rejects_forbidden_top_level_template_fields( + client: TestClient, +) -> None: + for forbidden_field, value in (("enabled", True), ("name", "templated-name")): + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"][forbidden_field] = value # type: ignore[index] + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("field") == forbidden_field and err.get("code") == "forbidden_template_field" + for err in body.get("errors", []) + ) + + def test_render_control_template_rejects_legacy_flat_format(client: TestClient) -> None: payload = _template_payload() payload["template"] = deepcopy(payload["template"]) @@ -341,3 +520,22 @@ def test_render_control_template_rejects_invalid_parameter_name_at_api_boundary( assert response.status_code == 422 assert response.json()["error_code"] == "VALIDATION_ERROR" + + +def test_render_control_template_keeps_non_parameterized_errors_on_rendered_fields( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"]["action"]["decision"] = "block" # type: ignore[index] + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("field") == "action.decision" + and err.get("rendered_field") == "action.decision" + for err in body.get("errors", []) + ) From 0773358f96096dbbef0c03113704a56f60111673 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 22:30:27 -0700 Subject: [PATCH 05/25] test: add template control edge case coverage --- server/tests/test_control_templates.py | 325 +++++++++++++++++++++++-- 1 file changed, 301 insertions(+), 24 deletions(-) diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 6c878239..03c0427a 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -90,7 +90,61 @@ def _defaults_only_template_payload() -> dict[str, object]: } -def _raw_control_payload(pattern: str = "raw") -> dict[str, object]: +def _case_sensitive_template_payload( + *, + values: list[str] | None = None, + case_sensitive: bool | None = None, + action: str = "deny", +) -> dict[str, object]: + template_values: dict[str, object] = {} + if values is not None: + template_values["values"] = values + if case_sensitive is not None: + template_values["case_sensitive"] = case_sensitive + + return { + "template": { + "description": "Case sensitivity template", + "parameters": { + "values": { + "type": "string_list", + "label": "Values", + "required": False, + "default": ["HELLO"], + }, + "case_sensitive": { + "type": "boolean", + "label": "Case Sensitive", + "required": False, + "default": True, + }, + }, + "definition_template": { + "description": "Case sensitivity control", + "execution": "server", + "scope": { + "step_names": ["templated-step"], + "stages": ["pre"], + }, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "list", + "config": { + "values": {"$param": "values"}, + "match_mode": "exact", + "case_sensitive": {"$param": "case_sensitive"}, + }, + }, + }, + "action": {"decision": action}, + }, + }, + "template_values": template_values, + } + + +def _raw_control_payload(pattern: str = "raw", *, action: str = "deny") -> dict[str, object]: return { "description": "Raw control", "enabled": True, @@ -103,7 +157,7 @@ def _raw_control_payload(pattern: str = "raw") -> dict[str, object]: "config": {"pattern": pattern}, }, }, - "action": {"decision": "deny"}, + "action": {"decision": action}, } @@ -112,13 +166,18 @@ def _create_template_control(client: TestClient) -> int: return control_id -def _create_template_control_with_name(client: TestClient) -> tuple[int, str]: - control_name = f"template-control-{uuid.uuid4()}" +def _create_template_control_with_name( + client: TestClient, + payload: dict[str, object] | None = None, + *, + name_prefix: str = "template-control", +) -> tuple[int, str]: + control_name = f"{name_prefix}-{uuid.uuid4()}" response = client.put( "/api/v1/controls", json={ "name": control_name, - "data": _template_payload(), + "data": payload or _template_payload(), }, ) assert response.status_code == 200, response.text @@ -130,14 +189,8 @@ def _assign_control_to_agent( control_id: int, *, agent_name: str | None = None, + via_policy: bool = True, ) -> str: - policy_response = client.put("/api/v1/policies", json={"name": f"policy-{uuid.uuid4()}"}) - assert policy_response.status_code == 200, policy_response.text - policy_id = policy_response.json()["policy_id"] - - add_control_response = client.post(f"/api/v1/policies/{policy_id}/controls/{control_id}") - assert add_control_response.status_code == 200, add_control_response.text - effective_agent_name = agent_name or f"template-agent-{uuid.uuid4().hex[:12]}" init_response = client.post( "/api/v1/agents/initAgent", @@ -145,8 +198,20 @@ def _assign_control_to_agent( ) assert init_response.status_code == 200, init_response.text - assign_response = client.post(f"/api/v1/agents/{effective_agent_name}/policy/{policy_id}") - assert assign_response.status_code == 200, assign_response.text + if via_policy: + policy_response = client.put("/api/v1/policies", json={"name": f"policy-{uuid.uuid4()}"}) + assert policy_response.status_code == 200, policy_response.text + policy_id = policy_response.json()["policy_id"] + + add_control_response = client.post(f"/api/v1/policies/{policy_id}/controls/{control_id}") + assert add_control_response.status_code == 200, add_control_response.text + + assign_response = client.post(f"/api/v1/agents/{effective_agent_name}/policy/{policy_id}") + assert assign_response.status_code == 200, assign_response.text + else: + assign_response = client.post(f"/api/v1/agents/{effective_agent_name}/controls/{control_id}") + assert assign_response.status_code == 200, assign_response.text + return effective_agent_name @@ -162,6 +227,23 @@ def _create_raw_control(client: TestClient) -> int: return response.json()["control_id"] +def _evaluate_step( + client: TestClient, + agent_name: str, + *, + step_name: str, + input_value: str, + step_type: str = "llm", + stage: str = "pre", +): + request = EvaluationRequest( + agent_name=agent_name, + step=Step(type=step_type, name=step_name, input=input_value, output=None), + stage=stage, + ) + return client.post("/api/v1/evaluation", json=request.model_dump(mode="json")) + + def test_render_control_template_preview_returns_rendered_control(client: TestClient) -> None: response = client.post("/api/v1/control-templates/render", json=_template_payload()) @@ -215,27 +297,222 @@ def test_template_backed_control_evaluates_after_policy_attachment(client: TestC control_id, control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) - safe_request = EvaluationRequest( - agent_name=agent_name, - step=Step(type="llm", name="other-step", input="hello", output=None), - stage="pre", + safe_response = _evaluate_step( + client, + agent_name, + step_name="other-step", + input_value="hello", ) - safe_response = client.post("/api/v1/evaluation", json=safe_request.model_dump(mode="json")) assert safe_response.status_code == 200, safe_response.text assert safe_response.json()["is_safe"] is True - deny_request = EvaluationRequest( - agent_name=agent_name, - step=Step(type="llm", name="templated-step", input="hello", output=None), - stage="pre", + deny_response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", ) - deny_response = client.post("/api/v1/evaluation", json=deny_request.model_dump(mode="json")) assert deny_response.status_code == 200, deny_response.text body = deny_response.json() assert body["is_safe"] is False assert body["matches"][0]["control_name"] == control_name +def test_template_backed_control_can_be_disabled_and_reenabled_in_evaluation( + client: TestClient, +) -> None: + control_id, control_name = _create_template_control_with_name(client) + agent_name = _assign_control_to_agent(client, control_id) + + initial_response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert initial_response.status_code == 200, initial_response.text + assert initial_response.json()["is_safe"] is False + + disable_response = client.patch(f"/api/v1/controls/{control_id}", json={"enabled": False}) + assert disable_response.status_code == 200, disable_response.text + + disabled_eval = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert disabled_eval.status_code == 200, disabled_eval.text + assert disabled_eval.json()["is_safe"] is True + + enable_response = client.patch(f"/api/v1/controls/{control_id}", json={"enabled": True}) + assert enable_response.status_code == 200, enable_response.text + + reenabled_eval = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert reenabled_eval.status_code == 200, reenabled_eval.text + body = reenabled_eval.json() + assert body["is_safe"] is False + assert body["matches"][0]["control_name"] == control_name + + +def test_template_backed_control_update_changes_scope_behavior(client: TestClient) -> None: + control_id, control_name = _create_template_control_with_name(client) + agent_name = _assign_control_to_agent(client, control_id) + + initial_eval = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert initial_eval.status_code == 200, initial_eval.text + assert initial_eval.json()["is_safe"] is False + + updated_payload = _template_payload() + updated_payload["template_values"] = { + "pattern": "hello", + "step_name": "updated-step", + } + update_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": updated_payload}, + ) + assert update_response.status_code == 200, update_response.text + + old_scope_eval = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert old_scope_eval.status_code == 200, old_scope_eval.text + assert old_scope_eval.json()["is_safe"] is True + + updated_scope_eval = _evaluate_step( + client, + agent_name, + step_name="updated-step", + input_value="hello", + ) + assert updated_scope_eval.status_code == 200, updated_scope_eval.text + body = updated_scope_eval.json() + assert body["is_safe"] is False + assert body["matches"][0]["control_name"] == control_name + + +def test_template_backed_control_supports_direct_agent_attachment(client: TestClient) -> None: + control_id, control_name = _create_template_control_with_name(client) + agent_name = _assign_control_to_agent(client, control_id, via_policy=False) + + response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["is_safe"] is False + assert body["matches"][0]["control_name"] == control_name + + +def test_template_backed_control_preserves_falsey_values_and_uses_them_in_behavior( + client: TestClient, +) -> None: + payload = _case_sensitive_template_payload(values=[], case_sensitive=False) + control_id, control_name = _create_template_control_with_name( + client, + payload, + name_prefix="falsey-template-control", + ) + agent_name = _assign_control_to_agent(client, control_id) + + get_response = client.get(f"/api/v1/controls/{control_id}/data") + assert get_response.status_code == 200, get_response.text + data = get_response.json()["data"] + assert data["template_values"] == { + "values": [], + "case_sensitive": False, + } + + non_applicable_eval = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert non_applicable_eval.status_code == 200, non_applicable_eval.text + assert non_applicable_eval.json()["is_safe"] is True + + updated_payload = _case_sensitive_template_payload( + values=["HELLO"], + case_sensitive=False, + ) + update_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": updated_payload}, + ) + assert update_response.status_code == 200, update_response.text + + deny_eval = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert deny_eval.status_code == 200, deny_eval.text + body = deny_eval.json() + assert body["is_safe"] is False + assert body["matches"][0]["control_name"] == control_name + + +def test_mixed_raw_and_template_backed_controls_obey_deny_precedence( + client: TestClient, +) -> None: + template_control_id, template_control_name = _create_template_control_with_name(client) + agent_name = _assign_control_to_agent(client, template_control_id) + + policy_response = client.get(f"/api/v1/agents/{agent_name}/policy") + assert policy_response.status_code == 200, policy_response.text + policy_id = policy_response.json()["policy_id"] + + raw_warn_name = f"raw-warn-{uuid.uuid4()}" + raw_warn_response = client.put( + "/api/v1/controls", + json={ + "name": raw_warn_name, + "data": _raw_control_payload("hello", action="warn"), + }, + ) + assert raw_warn_response.status_code == 200, raw_warn_response.text + raw_warn_control_id = raw_warn_response.json()["control_id"] + + add_control_response = client.post(f"/api/v1/policies/{policy_id}/controls/{raw_warn_control_id}") + assert add_control_response.status_code == 200, add_control_response.text + + response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["is_safe"] is False + assert len(body["matches"]) == 2 + names = {match["control_name"] for match in body["matches"]} + actions = {match["action"] for match in body["matches"]} + assert names == {template_control_name, raw_warn_name} + assert actions == {"deny", "warn"} + + def test_raw_control_can_be_replaced_with_template_backed_control(client: TestClient) -> None: control_id = _create_raw_control(client) From 8bc76a0d3b71eb579805d08769a51872a129d274 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 22:44:13 -0700 Subject: [PATCH 06/25] fix: tighten template control validation ergonomics --- models/src/agent_control_models/controls.py | 9 +- models/src/agent_control_models/server.py | 20 ++++ models/tests/test_control_templates.py | 50 ++++++++- sdks/python/tests/test_controls_api.py | 102 ++++++++++++++++++ .../endpoints/controls.py | 8 +- .../services/control_templates.py | 20 +++- server/tests/test_control_templates.py | 23 ++++ 7 files changed, 217 insertions(+), 15 deletions(-) diff --git a/models/src/agent_control_models/controls.py b/models/src/agent_control_models/controls.py index ca6b0a90..bdc10658 100644 --- a/models/src/agent_control_models/controls.py +++ b/models/src/agent_control_models/controls.py @@ -5,7 +5,7 @@ import re from collections.abc import Iterator from dataclasses import dataclass -from typing import Annotated, Any, Literal, Self, cast +from typing import Annotated, Any, Literal, Self from uuid import uuid4 import re2 @@ -871,13 +871,6 @@ def observability_identity(self) -> ControlObservabilityIdentity: all_selector_paths=all_selector_paths, ) - def to_control_definition(self) -> ControlDefinition: - """Promote a runtime control to the full public model when needed.""" - return ControlDefinition.model_validate( - cast(dict[str, JSONValue], self.model_dump(mode="python")) - ) - - class EvaluatorResult(BaseModel): """Result from a control evaluator. diff --git a/models/src/agent_control_models/server.py b/models/src/agent_control_models/server.py index a0e099de..bfe5c015 100644 --- a/models/src/agent_control_models/server.py +++ b/models/src/agent_control_models/server.py @@ -16,6 +16,20 @@ def _strip_slug_name(v: str) -> str: _CONTROL_DEFINITION_ADAPTER = TypeAdapter(ControlDefinition) _TEMPLATE_CONTROL_INPUT_ADAPTER = TypeAdapter(TemplateControlInput) +_RAW_CONTROL_INPUT_FIELDS = frozenset( + { + "description", + "enabled", + "execution", + "scope", + "condition", + "action", + "tags", + # Legacy flat leaf fields still accepted for raw controls. + "selector", + "evaluator", + } +) def _parse_control_input(v: Any) -> Any: @@ -30,6 +44,12 @@ def _parse_control_input(v: Any) -> Any: return v if v.get("template") is not None: + mixed_fields = sorted(field for field in v if field in _RAW_CONTROL_INPUT_FIELDS) + if mixed_fields: + raise ValueError( + "Template-backed control input cannot mix template fields with rendered control " + f"fields. Remove raw fields: {', '.join(mixed_fields)}." + ) return _TEMPLATE_CONTROL_INPUT_ADAPTER.validate_python(v) return _CONTROL_DEFINITION_ADAPTER.validate_python(v) diff --git a/models/tests/test_control_templates.py b/models/tests/test_control_templates.py index d2a7725c..50b2c903 100644 --- a/models/tests/test_control_templates.py +++ b/models/tests/test_control_templates.py @@ -58,6 +58,28 @@ def test_control_definition_requires_template_fields_together() -> None: ) +def test_control_definition_rejects_template_values_without_template() -> None: + with pytest.raises( + ValidationError, + match="template and template_values must both be present or both absent", + ): + ControlDefinition.model_validate( + { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "ok"}, + }, + }, + "action": {"decision": "deny"}, + "template_values": {"pattern": "hello"}, + } + ) + + def test_template_definition_rejects_invalid_parameter_name() -> None: with pytest.raises( ValidationError, @@ -91,7 +113,13 @@ def test_create_control_request_parses_template_payload_as_template_input() -> N def test_create_control_request_rejects_mixed_raw_and_template_payload() -> None: - with pytest.raises(ValidationError): + with pytest.raises( + ValidationError, + match=( + "Template-backed control input cannot mix template fields with rendered " + "control fields" + ), + ): CreateControlRequest.model_validate( { "name": "template-control", @@ -130,6 +158,26 @@ def test_control_definition_can_round_trip_to_template_control_input() -> None: assert template_input.template_values == {"pattern": "hello"} +def test_control_definition_to_template_control_input_rejects_raw_control() -> None: + control = ControlDefinition.model_validate( + { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "hello"}, + }, + }, + "action": {"decision": "deny"}, + } + ) + + with pytest.raises(ValueError, match="not template-backed"): + control.to_template_control_input() + + def test_control_definition_runtime_ignores_template_metadata() -> None: runtime_control = ControlDefinitionRuntime.model_validate( { diff --git a/sdks/python/tests/test_controls_api.py b/sdks/python/tests/test_controls_api.py index e19cdaed..d822b01e 100644 --- a/sdks/python/tests/test_controls_api.py +++ b/sdks/python/tests/test_controls_api.py @@ -109,6 +109,90 @@ async def test_render_control_template_calls_preview_endpoint() -> None: ) +@pytest.mark.asyncio +async def test_validate_control_data_accepts_template_control_input() -> None: + response = Mock() + response.raise_for_status = Mock() + response.json = Mock(return_value={"success": True}) + client = SimpleNamespace(http_client=SimpleNamespace(post=AsyncMock(return_value=response))) + template_input = TemplateControlInput.model_validate( + { + "template": { + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + } + }, + "definition_template": { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {"pattern": "hello"}, + } + ) + + await agent_control.controls.validate_control_data(client, template_input) + + client.http_client.post.assert_awaited_once() + _, kwargs = client.http_client.post.await_args + assert kwargs["json"]["data"]["template_values"]["pattern"] == "hello" + assert kwargs["json"] == { + "data": { + "template": kwargs["json"]["data"]["template"], + "template_values": {"pattern": "hello"}, + } + } + + +@pytest.mark.asyncio +async def test_set_control_data_accepts_template_control_input() -> None: + response = Mock() + response.raise_for_status = Mock() + response.json = Mock(return_value={"success": True}) + client = SimpleNamespace(http_client=SimpleNamespace(put=AsyncMock(return_value=response))) + template_input = TemplateControlInput.model_validate( + { + "template": { + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + } + }, + "definition_template": { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {"pattern": "hello"}, + } + ) + + await agent_control.controls.set_control_data(client, 123, template_input) + + client.http_client.put.assert_awaited_once() + _, kwargs = client.http_client.put.await_args + assert kwargs["json"]["data"]["template_values"]["pattern"] == "hello" + + def test_to_template_control_input_reshapes_stored_control_data() -> None: template_input = agent_control.controls.to_template_control_input( { @@ -148,3 +232,21 @@ def test_to_template_control_input_reshapes_stored_control_data() -> None: assert isinstance(template_input, TemplateControlInput) assert template_input.template_values == {"pattern": "hello"} + + +def test_to_template_control_input_rejects_raw_control_data() -> None: + with pytest.raises(ValueError, match="not template-backed"): + agent_control.controls.to_template_control_input( + { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": "hello"}, + }, + }, + "action": {"decision": "deny"}, + } + ) diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index a8a0cf3e..41165132 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -528,11 +528,9 @@ async def set_control_data( current_template_backed = _is_template_backed_payload(control.data) if isinstance(request.data, TemplateControlInput): current_enabled = True - if control.data: - try: - current_enabled = ControlDefinition.model_validate(control.data).enabled - except ValidationError: - current_enabled = True + if isinstance(control.data, dict): + raw_enabled = control.data.get("enabled", True) + current_enabled = raw_enabled if type(raw_enabled) is bool else True control_def = await _render_and_validate_template_input( request.data, db=db, diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index 5dc4a1d6..bad54bd6 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -297,9 +297,27 @@ def _render_json_value( message=f"Template references undefined parameter '{parameter_name}'.", ) + parameter_definition = template.parameters[parameter_name] + if ( + not parameter_definition.required + and _parameter_default(parameter_definition) is _TEMPLATE_VALUE_MISSING + ): + raise _render_error( + detail=( + f"Template parameter '{parameter_name}' is optional but referenced in " + "definition_template without a default value" + ), + field=f"template.parameters.{parameter_name}", + code="optional_referenced_parameter_requires_default", + message=( + f"Optional template parameter '{parameter_definition.label}' is " + "referenced in the template and must define a default value or be " + "marked required." + ), + ) + resolved_value = resolved_values[parameter_name] if resolved_value is _TEMPLATE_VALUE_MISSING: - parameter_definition = template.parameters[parameter_name] raise _parameter_error( parameter_name, parameter_definition, diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 03c0427a..527cc6fe 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -669,6 +669,29 @@ def test_render_control_template_maps_invalid_regex_parameter(client: TestClient ) +def test_render_control_template_rejects_optional_referenced_parameter_without_default( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["parameters"]["step_name"] = { # type: ignore[index] + "type": "string", + "label": "Step Name", + "required": False, + } + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("field") == "template.parameters.step_name" + and err.get("code") == "optional_referenced_parameter_requires_default" + for err in body.get("errors", []) + ) + + def test_render_control_template_rejects_malformed_param_binding(client: TestClient) -> None: payload = _template_payload() payload["template"] = deepcopy(payload["template"]) From d1ee234a94f69cc66b8695038498b82fa04ac292 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:00:30 -0700 Subject: [PATCH 07/25] refactor: harden template control runtime helpers --- engine/src/agent_control_engine/core.py | 28 ++- models/src/agent_control_models/controls.py | 166 ++++++++++-------- models/tests/test_control_templates.py | 33 ++++ .../endpoints/controls.py | 36 +--- .../endpoints/evaluation.py | 17 +- .../services/condition_traversal.py | 35 ++++ .../services/control_templates.py | 33 +--- server/tests/test_control_templates.py | 21 +++ 8 files changed, 218 insertions(+), 151 deletions(-) create mode 100644 server/src/agent_control_server/services/condition_traversal.py diff --git a/engine/src/agent_control_engine/core.py b/engine/src/agent_control_engine/core.py index 2600b7d7..99c2273b 100644 --- a/engine/src/agent_control_engine/core.py +++ b/engine/src/agent_control_engine/core.py @@ -15,9 +15,9 @@ from agent_control_evaluators import get_evaluator_instance from agent_control_models import ( ConditionNode, - ControlDefinition, - ControlDefinitionRuntime, + ControlAction, ControlMatch, + ControlScope, EvaluationRequest, EvaluationResponse, EvaluatorResult, @@ -43,12 +43,30 @@ def _compile_regex(pattern: str) -> Any: return re2.compile(pattern) +class ControlDefinitionLike(Protocol): + """Runtime control shape required by the engine.""" + + enabled: bool + execution: Literal["server", "sdk"] + scope: ControlScope + condition: ConditionNode + action: ControlAction + + class ControlWithIdentity(Protocol): """Protocol for a control with identity information.""" - id: int - name: str - control: ControlDefinition | ControlDefinitionRuntime + @property + def id(self) -> int: + """Database identity for the control.""" + + @property + def name(self) -> str: + """Human-readable name for the control.""" + + @property + def control(self) -> ControlDefinitionLike: + """Runtime control payload used during evaluation.""" @dataclass diff --git a/models/src/agent_control_models/controls.py b/models/src/agent_control_models/controls.py index bdc10658..65993fc7 100644 --- a/models/src/agent_control_models/controls.py +++ b/models/src/agent_control_models/controls.py @@ -268,6 +268,8 @@ class SteeringContext(BaseModel): type JsonValue = JSONValue _TEMPLATE_PARAMETER_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") +MAX_TEMPLATE_DEFINITION_DEPTH = 12 +MAX_TEMPLATE_DEFINITION_NODES = 1000 def _validate_re2_value(value: str, *, field_name: str) -> str: @@ -279,6 +281,35 @@ def _validate_re2_value(value: str, *, field_name: str) -> str: return value +def _validate_template_definition_structure(value: JsonValue) -> JsonValue: + """Validate template definition nesting and overall size.""" + stack: list[tuple[JsonValue, int]] = [(value, 1)] + node_count = 0 + + while stack: + node, depth = stack.pop() + node_count += 1 + + if depth > MAX_TEMPLATE_DEFINITION_DEPTH: + raise ValueError( + "definition_template nesting depth exceeds maximum of " + f"{MAX_TEMPLATE_DEFINITION_DEPTH}" + ) + + if node_count > MAX_TEMPLATE_DEFINITION_NODES: + raise ValueError( + "definition_template size exceeds maximum of " + f"{MAX_TEMPLATE_DEFINITION_NODES} JSON nodes" + ) + + if isinstance(node, dict): + stack.extend((nested_value, depth + 1) for nested_value in node.values()) + elif isinstance(node, list): + stack.extend((nested_value, depth + 1) for nested_value in node) + + return value + + class TemplateParameterBase(BaseModel): """Base definition for a template parameter.""" @@ -416,6 +447,12 @@ def validate_parameter_names( ) return value + @field_validator("definition_template") + @classmethod + def validate_definition_template_structure(cls, value: JsonValue) -> JsonValue: + """Limit template nesting and size to keep rendering bounded.""" + return _validate_template_definition_structure(value) + class TemplateControlInput(BaseModel): """Template-backed input payload for control create/update requests.""" @@ -626,7 +663,56 @@ def leaf_parts(self) -> ConditionLeafParts | None: ConditionNode.model_rebuild() -class ControlDefinition(BaseModel): +def _build_observability_identity( + condition: ConditionNode, +) -> ControlObservabilityIdentity: + """Build a stable selector/evaluator identity for a condition tree.""" + all_evaluators: list[str] = [] + all_selector_paths: list[str] = [] + seen_evaluators: set[str] = set() + seen_selector_paths: set[str] = set() + leaf_count = 0 + + for selector, evaluator in condition.iter_leaf_parts(): + leaf_count += 1 + selector_path = selector.path or "*" + + if evaluator.name not in seen_evaluators: + seen_evaluators.add(evaluator.name) + all_evaluators.append(evaluator.name) + + if selector_path not in seen_selector_paths: + seen_selector_paths.add(selector_path) + all_selector_paths.append(selector_path) + + return ControlObservabilityIdentity( + selector_path=all_selector_paths[0] if all_selector_paths else None, + evaluator_name=all_evaluators[0] if all_evaluators else None, + leaf_count=leaf_count, + all_evaluators=all_evaluators, + all_selector_paths=all_selector_paths, + ) + + +class _ConditionBackedControlMixin: + """Shared helpers for control models backed by a condition tree.""" + + condition: ConditionNode + + def iter_condition_leaves(self) -> Iterator[ConditionNode]: + """Yield leaf conditions in evaluation order.""" + yield from self.condition.iter_leaves() + + def iter_condition_leaf_parts(self) -> Iterator[ConditionLeafParts]: + """Yield leaf selector/evaluator pairs in evaluation order.""" + yield from self.condition.iter_leaf_parts() + + def observability_identity(self) -> ControlObservabilityIdentity: + """Return a deterministic representative identity for observability.""" + return _build_observability_identity(self.condition) + + +class ControlDefinition(_ConditionBackedControlMixin, BaseModel): """A control definition to evaluate agent interactions. This model contains only the logic and configuration. @@ -725,53 +811,12 @@ def to_template_control_input(self) -> TemplateControlInput: template_values=dict(self.template_values), ) - def iter_condition_leaves(self) -> Iterator[ConditionNode]: - """Yield leaf conditions in evaluation order.""" - yield from self.condition.iter_leaves() - - def iter_condition_leaf_parts(self) -> Iterator[ConditionLeafParts]: - """Yield leaf selector/evaluator pairs in evaluation order.""" - yield from self.condition.iter_leaf_parts() - def primary_leaf(self) -> ConditionNode | None: """Return the single leaf node when the whole condition is just one leaf.""" if self.condition.is_leaf(): return self.condition return None - def observability_identity(self) -> ControlObservabilityIdentity: - """Return a deterministic representative identity for observability. - - The representative selector/evaluator comes from the first leaf in - evaluation order so composite trees still populate top-level event - dimensions. The full ordered, deduped leaf context is also returned. - """ - all_evaluators: list[str] = [] - all_selector_paths: list[str] = [] - seen_evaluators: set[str] = set() - seen_selector_paths: set[str] = set() - leaf_count = 0 - - for selector, evaluator in self.iter_condition_leaf_parts(): - leaf_count += 1 - selector_path = selector.path or "*" - - if evaluator.name not in seen_evaluators: - seen_evaluators.add(evaluator.name) - all_evaluators.append(evaluator.name) - - if selector_path not in seen_selector_paths: - seen_selector_paths.add(selector_path) - all_selector_paths.append(selector_path) - - return ControlObservabilityIdentity( - selector_path=all_selector_paths[0] if all_selector_paths else None, - evaluator_name=all_evaluators[0] if all_evaluators else None, - leaf_count=leaf_count, - all_evaluators=all_evaluators, - all_selector_paths=all_selector_paths, - ) - model_config = { "json_schema_extra": { "examples": [ @@ -799,7 +844,7 @@ def observability_identity(self) -> ControlObservabilityIdentity: } -class ControlDefinitionRuntime(BaseModel): +class ControlDefinitionRuntime(_ConditionBackedControlMixin, BaseModel): """Slim runtime control model that ignores template authoring metadata.""" model_config = ConfigDict(extra="ignore") @@ -835,41 +880,6 @@ def validate_condition_constraints(self) -> Self: _validate_common_control_constraints(self.condition, self.action) return self - def iter_condition_leaves(self) -> Iterator[ConditionNode]: - """Yield leaf conditions in evaluation order.""" - yield from self.condition.iter_leaves() - - def iter_condition_leaf_parts(self) -> Iterator[ConditionLeafParts]: - """Yield leaf selector/evaluator pairs in evaluation order.""" - yield from self.condition.iter_leaf_parts() - - def observability_identity(self) -> ControlObservabilityIdentity: - """Return a deterministic representative identity for observability.""" - all_evaluators: list[str] = [] - all_selector_paths: list[str] = [] - seen_evaluators: set[str] = set() - seen_selector_paths: set[str] = set() - leaf_count = 0 - - for selector, evaluator in self.iter_condition_leaf_parts(): - leaf_count += 1 - selector_path = selector.path or "*" - - if evaluator.name not in seen_evaluators: - seen_evaluators.add(evaluator.name) - all_evaluators.append(evaluator.name) - - if selector_path not in seen_selector_paths: - seen_selector_paths.add(selector_path) - all_selector_paths.append(selector_path) - - return ControlObservabilityIdentity( - selector_path=all_selector_paths[0] if all_selector_paths else None, - evaluator_name=all_evaluators[0] if all_evaluators else None, - leaf_count=leaf_count, - all_evaluators=all_evaluators, - all_selector_paths=all_selector_paths, - ) class EvaluatorResult(BaseModel): """Result from a control evaluator. diff --git a/models/tests/test_control_templates.py b/models/tests/test_control_templates.py index 50b2c903..517a7624 100644 --- a/models/tests/test_control_templates.py +++ b/models/tests/test_control_templates.py @@ -36,6 +36,13 @@ } +def _nested_template_value(depth: int) -> object: + value: object = "leaf" + for _ in range(depth): + value = {"nested": value} + return value + + def test_control_definition_requires_template_fields_together() -> None: with pytest.raises( ValidationError, @@ -98,6 +105,32 @@ def test_template_definition_rejects_invalid_parameter_name() -> None: ) +def test_template_definition_rejects_excessive_nesting() -> None: + with pytest.raises( + ValidationError, + match="definition_template nesting depth exceeds maximum", + ): + TemplateDefinition.model_validate( + { + "parameters": {}, + "definition_template": _nested_template_value(12), + } + ) + + +def test_template_definition_rejects_excessive_size() -> None: + with pytest.raises( + ValidationError, + match="definition_template size exceeds maximum", + ): + TemplateDefinition.model_validate( + { + "parameters": {}, + "definition_template": list(range(1001)), + } + ) + + def test_create_control_request_parses_template_payload_as_template_input() -> None: request = CreateControlRequest.model_validate( { diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 41165132..ce0ae9d9 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -1,7 +1,5 @@ -from collections.abc import Iterator - from agent_control_engine import list_evaluators -from agent_control_models import ConditionNode, ControlDefinition, TemplateControlInput +from agent_control_models import ControlDefinition, TemplateControlInput from agent_control_models.errors import ErrorCode, ValidationErrorItem from agent_control_models.server import ( AgentRef, @@ -39,6 +37,7 @@ ) from ..logging_utils import get_logger from ..models import Agent, AgentData, Control, agent_controls, agent_policies, policy_controls +from ..services.condition_traversal import iter_condition_leaves_with_paths from ..services.control_definitions import parse_control_definition_or_api_error from ..services.control_templates import remap_template_api_error, render_template_control_input from ..services.evaluator_utils import ( @@ -59,32 +58,6 @@ template_router = APIRouter(prefix="/control-templates", tags=["controls"]) _logger = get_logger(__name__) - - -def _iter_condition_leaves( - node: ConditionNode, - *, - path: str = "data.condition", -) -> Iterator[tuple[str, ConditionNode]]: - """Yield each leaf condition with its dot/bracket field path.""" - if node.is_leaf(): - yield path, node - return - - if node.and_ is not None: - for index, child in enumerate(node.and_): - yield from _iter_condition_leaves(child, path=f"{path}.and[{index}]") - return - - if node.or_ is not None: - for index, child in enumerate(node.or_): - yield from _iter_condition_leaves(child, path=f"{path}.or[{index}]") - return - - if node.not_ is not None: - yield from _iter_condition_leaves(node.not_, path=f"{path}.not") - - def _serialize_control_definition(control_def: ControlDefinition) -> dict[str, object]: """Serialize control data for storage while omitting null scope fields.""" data_json = control_def.model_dump( @@ -130,7 +103,10 @@ async def _validate_control_definition( """Validate evaluator config for a control definition.""" available_evaluators = list_evaluators() agent_data_by_name: dict[str, AgentData] = {} - for field_prefix, leaf in _iter_condition_leaves(control_def.condition): + for field_prefix, leaf in iter_condition_leaves_with_paths( + control_def.condition, + path="data.condition", + ): leaf_parts = leaf.leaf_parts() if leaf_parts is None: continue diff --git a/server/src/agent_control_server/endpoints/evaluation.py b/server/src/agent_control_server/endpoints/evaluation.py index ddade004..ba5f85a9 100644 --- a/server/src/agent_control_server/endpoints/evaluation.py +++ b/server/src/agent_control_server/endpoints/evaluation.py @@ -1,12 +1,12 @@ """Evaluation analysis endpoints.""" import time +from dataclasses import dataclass from datetime import UTC, datetime from typing import Literal from agent_control_engine.core import ControlEngine from agent_control_models import ( - ControlDefinition, ControlDefinitionRuntime, ControlExecutionEvent, ControlMatch, @@ -41,18 +41,13 @@ SAFE_ENGINE_VALIDATION_MESSAGE = "Invalid evaluation request or control configuration." +@dataclass class ControlAdapter: """Adapts API Control to Engine ControlWithIdentity protocol.""" - def __init__( - self, - id: int, - name: str, - control: ControlDefinition | ControlDefinitionRuntime, - ): - self.id = id - self.name = name - self.control = control + id: int + name: str + control: ControlDefinitionRuntime def _sanitize_evaluator_error(error_message: str) -> str: @@ -133,7 +128,7 @@ def _sanitize_evaluation_response(response: EvaluationResponse) -> EvaluationRes def _observability_metadata( - control_def: ControlDefinition, + control_def: ControlDefinitionRuntime, ) -> tuple[str | None, str | None, dict[str, object]]: """Return representative event fields plus full composite context.""" identity = control_def.observability_identity() diff --git a/server/src/agent_control_server/services/condition_traversal.py b/server/src/agent_control_server/services/condition_traversal.py new file mode 100644 index 00000000..9e9ab6b8 --- /dev/null +++ b/server/src/agent_control_server/services/condition_traversal.py @@ -0,0 +1,35 @@ +"""Shared condition-tree traversal helpers.""" + +from collections.abc import Iterator + +from agent_control_models import ConditionNode + + +def iter_condition_leaves_with_paths( + node: ConditionNode, + *, + path: str, +) -> Iterator[tuple[str, ConditionNode]]: + """Yield each leaf condition with its dotted/bracketed field path.""" + if node.is_leaf(): + yield path, node + return + + if node.and_ is not None: + for index, child in enumerate(node.and_): + yield from iter_condition_leaves_with_paths( + child, + path=f"{path}.and[{index}]", + ) + return + + if node.or_ is not None: + for index, child in enumerate(node.or_): + yield from iter_condition_leaves_with_paths( + child, + path=f"{path}.or[{index}]", + ) + return + + if node.not_ is not None: + yield from iter_condition_leaves_with_paths(node.not_, path=f"{path}.not") diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index bad54bd6..2d153a77 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -2,13 +2,12 @@ from __future__ import annotations -from collections.abc import Iterator, Mapping +from collections.abc import Mapping from dataclasses import dataclass from typing import cast import re2 from agent_control_models import ( - ConditionNode, ControlDefinition, EnumTemplateParameter, JsonValue, @@ -21,6 +20,7 @@ from pydantic import ValidationError from ..errors import APIValidationError +from .condition_traversal import iter_condition_leaves_with_paths from .validation_paths import format_field_path _TEMPLATE_VALUE_MISSING = object() @@ -439,30 +439,6 @@ def remap_template_api_error( ) -def _iter_condition_leaves( - node: ConditionNode, - *, - path: str = "condition", -) -> Iterator[tuple[str, ConditionNode]]: - """Yield each leaf condition with a dotted/bracketed field path.""" - if node.is_leaf(): - yield path, node - return - - if node.and_ is not None: - for index, child in enumerate(node.and_): - yield from _iter_condition_leaves(child, path=f"{path}.and[{index}]") - return - - if node.or_ is not None: - for index, child in enumerate(node.or_): - yield from _iter_condition_leaves(child, path=f"{path}.or[{index}]") - return - - if node.not_ is not None: - yield from _iter_condition_leaves(node.not_, path=f"{path}.not") - - def _reject_agent_scoped_evaluators( control: ControlDefinition, *, @@ -470,7 +446,10 @@ def _reject_agent_scoped_evaluators( template: TemplateDefinition, ) -> None: """Reject agent-scoped evaluator references in v1 templates.""" - for field_prefix, leaf in _iter_condition_leaves(control.condition): + for field_prefix, leaf in iter_condition_leaves_with_paths( + control.condition, + path="condition", + ): leaf_parts = leaf.leaf_parts() if leaf_parts is None: continue diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 527cc6fe..793e4d26 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -161,6 +161,13 @@ def _raw_control_payload(pattern: str = "raw", *, action: str = "deny") -> dict[ } +def _nested_template_value(depth: int) -> object: + value: object = "leaf" + for _ in range(depth): + value = {"nested": value} + return value + + def _create_template_control(client: TestClient) -> int: control_id, _ = _create_template_control_with_name(client) return control_id @@ -279,6 +286,20 @@ def test_render_control_template_preview_uses_defaults_when_values_omitted( } +def test_render_control_template_preview_rejects_excessive_definition_nesting( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"]["definition_template"] = _nested_template_value(12) + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422, response.text + body = response.json() + assert body["detail"].startswith("Request validation failed") + assert "definition_template nesting depth exceeds maximum" in body["errors"][0]["message"] + + def test_create_template_backed_control_persists_template_metadata(client: TestClient) -> None: control_id = _create_template_control(client) From 9695c8207e67521f6d3893732629e96e0af650a6 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:10:06 -0700 Subject: [PATCH 08/25] test: cover remaining template control api cases --- server/tests/test_control_templates.py | 126 +++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 793e4d26..b35a8082 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -300,6 +300,20 @@ def test_render_control_template_preview_rejects_excessive_definition_nesting( assert "definition_template nesting depth exceeds maximum" in body["errors"][0]["message"] +def test_render_control_template_preview_rejects_excessive_definition_size( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"]["definition_template"] = list(range(1001)) + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422, response.text + body = response.json() + assert body["detail"].startswith("Request validation failed") + assert "definition_template size exceeds maximum" in body["errors"][0]["message"] + + def test_create_template_backed_control_persists_template_metadata(client: TestClient) -> None: control_id = _create_template_control(client) @@ -314,6 +328,31 @@ def test_create_template_backed_control_persists_template_metadata(client: TestC assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" +def test_create_template_backed_control_persists_resolved_defaults_when_values_omitted( + client: TestClient, +) -> None: + control_id, _ = _create_template_control_with_name( + client, + _defaults_only_template_payload(), + name_prefix="defaulted-template-control", + ) + + response = client.get(f"/api/v1/controls/{control_id}/data") + + assert response.status_code == 200, response.text + data = response.json()["data"] + assert data["template_values"] == { + "values": ["secret", "blocked"], + "logic": "any", + "case_sensitive": False, + } + assert data["condition"]["evaluator"]["config"] == { + "values": ["secret", "blocked"], + "logic": "any", + "case_sensitive": False, + } + + def test_template_backed_control_evaluates_after_policy_attachment(client: TestClient) -> None: control_id, control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) @@ -443,6 +482,34 @@ def test_template_backed_control_supports_direct_agent_attachment(client: TestCl assert body["matches"][0]["control_name"] == control_name +def test_template_backed_warn_control_evaluates_as_safe_with_warn_match( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"]["action"]["decision"] = "warn" # type: ignore[index] + control_id, control_name = _create_template_control_with_name( + client, + payload, + name_prefix="warn-template-control", + ) + agent_name = _assign_control_to_agent(client, control_id) + + response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["is_safe"] is True + assert len(body["matches"]) == 1 + assert body["matches"][0]["control_name"] == control_name + assert body["matches"][0]["action"] == "warn" + + def test_template_backed_control_preserves_falsey_values_and_uses_them_in_behavior( client: TestClient, ) -> None: @@ -615,6 +682,25 @@ def test_render_control_template_rejects_unknown_template_value_key(client: Test ) +def test_render_control_template_rejects_undefined_param_reference(client: TestClient) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["pattern"] = { # type: ignore[index] + "$param": "undefined_pattern", + } + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("code") == "undefined_parameter_reference" + and err.get("field") == "condition.evaluator.config.pattern" + for err in body.get("errors", []) + ) + + def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> None: control_id = _create_template_control(client) raw_payload = deepcopy( @@ -643,6 +729,46 @@ def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> N assert response.json()["error_code"] == "CONTROL_TEMPLATE_CONFLICT" +def test_create_control_rejects_mixed_raw_and_template_payload_at_api_boundary( + client: TestClient, +) -> None: + payload = _template_payload() + payload["execution"] = "server" + + response = client.put( + "/api/v1/controls", + json={ + "name": f"mixed-template-control-{uuid.uuid4()}", + "data": payload, + }, + ) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "VALIDATION_ERROR" + assert any( + "cannot mix template fields with rendered control fields" in err.get("message", "") + for err in body.get("errors", []) + ) + + +def test_validate_control_rejects_mixed_raw_and_template_payload_at_api_boundary( + client: TestClient, +) -> None: + payload = _template_payload() + payload["execution"] = "server" + + response = client.post("/api/v1/controls/validate", json={"data": payload}) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "VALIDATION_ERROR" + assert any( + "cannot mix template fields with rendered control fields" in err.get("message", "") + for err in body.get("errors", []) + ) + + def test_list_controls_includes_template_backed_flag_and_filter(client: TestClient) -> None: control_id = _create_template_control(client) From 0b4dc5d75f92a5b920f8ef756d270dfd0e4a8456 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:12:22 -0700 Subject: [PATCH 09/25] refactor: derive raw control parser fields --- models/src/agent_control_models/server.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/models/src/agent_control_models/server.py b/models/src/agent_control_models/server.py index bfe5c015..89c92ec1 100644 --- a/models/src/agent_control_models/server.py +++ b/models/src/agent_control_models/server.py @@ -16,15 +16,10 @@ def _strip_slug_name(v: str) -> str: _CONTROL_DEFINITION_ADAPTER = TypeAdapter(ControlDefinition) _TEMPLATE_CONTROL_INPUT_ADAPTER = TypeAdapter(TemplateControlInput) -_RAW_CONTROL_INPUT_FIELDS = frozenset( +_TEMPLATE_ONLY_CONTROL_FIELDS = frozenset({"template", "template_values"}) +_RAW_CONTROL_INPUT_FIELDS = frozenset(ControlDefinition.model_fields) - _TEMPLATE_ONLY_CONTROL_FIELDS +_RAW_CONTROL_INPUT_FIELDS = _RAW_CONTROL_INPUT_FIELDS.union( { - "description", - "enabled", - "execution", - "scope", - "condition", - "action", - "tags", # Legacy flat leaf fields still accepted for raw controls. "selector", "evaluator", From d344f542c6e64dbf2756382efbb18659a06f79eb Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:16:51 -0700 Subject: [PATCH 10/25] test: cover remaining template control server flows --- server/tests/test_control_templates.py | 81 ++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index b35a8082..b726d977 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -328,6 +328,24 @@ def test_create_template_backed_control_persists_template_metadata(client: TestC assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" +def test_get_control_returns_template_metadata_for_template_backed_control( + client: TestClient, +) -> None: + control_id, control_name = _create_template_control_with_name(client) + + response = client.get(f"/api/v1/controls/{control_id}") + + assert response.status_code == 200, response.text + body = response.json() + assert body["id"] == control_id + assert body["name"] == control_name + assert body["data"]["template"]["description"] == "Regex denial template" + assert body["data"]["template_values"] == { + "pattern": "hello", + "step_name": "templated-step", + } + + def test_create_template_backed_control_persists_resolved_defaults_when_values_omitted( client: TestClient, ) -> None: @@ -420,6 +438,59 @@ def test_template_backed_control_can_be_disabled_and_reenabled_in_evaluation( assert body["matches"][0]["control_name"] == control_name +def test_template_backed_control_rename_is_reflected_in_evaluation(client: TestClient) -> None: + control_id, _ = _create_template_control_with_name(client) + agent_name = _assign_control_to_agent(client, control_id) + renamed_control_name = f"renamed-template-control-{uuid.uuid4()}" + + patch_response = client.patch( + f"/api/v1/controls/{control_id}", + json={"name": renamed_control_name}, + ) + assert patch_response.status_code == 200, patch_response.text + assert patch_response.json()["name"] == renamed_control_name + + eval_response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + + assert eval_response.status_code == 200, eval_response.text + body = eval_response.json() + assert body["is_safe"] is False + assert body["matches"][0]["control_name"] == renamed_control_name + + +def test_template_backed_control_patch_updates_name_and_enabled_together( + client: TestClient, +) -> None: + control_id, _ = _create_template_control_with_name(client) + renamed_control_name = f"patched-template-control-{uuid.uuid4()}" + + patch_response = client.patch( + f"/api/v1/controls/{control_id}", + json={"name": renamed_control_name, "enabled": False}, + ) + + assert patch_response.status_code == 200, patch_response.text + body = patch_response.json() + assert body["name"] == renamed_control_name + assert body["enabled"] is False + + get_response = client.get(f"/api/v1/controls/{control_id}") + assert get_response.status_code == 200, get_response.text + get_body = get_response.json() + assert get_body["name"] == renamed_control_name + assert get_body["data"]["enabled"] is False + assert get_body["data"]["template"]["description"] == "Regex denial template" + assert get_body["data"]["template_values"] == { + "pattern": "hello", + "step_name": "templated-step", + } + + def test_template_backed_control_update_changes_scope_behavior(client: TestClient) -> None: control_id, control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) @@ -666,6 +737,16 @@ def test_template_validate_maps_missing_parameter_error(client: TestClient) -> N ) +def test_template_validate_succeeds_with_defaults_only_payload(client: TestClient) -> None: + response = client.post( + "/api/v1/controls/validate", + json={"data": _defaults_only_template_payload()}, + ) + + assert response.status_code == 200, response.text + assert response.json()["success"] is True + + def test_render_control_template_rejects_unknown_template_value_key(client: TestClient) -> None: payload = _template_payload() payload["template_values"] = {"pattern": "hello", "unknown": "value"} From 1a946386bc317207668a30a10944c7f6fe1906ab Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:26:38 -0700 Subject: [PATCH 11/25] fix: cover remaining template control edge cases --- .../services/control_templates.py | 47 +++--- server/tests/test_control_templates.py | 144 ++++++++++++++++++ 2 files changed, 171 insertions(+), 20 deletions(-) diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index 2d153a77..af93b416 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -489,30 +489,37 @@ def render_template_control_input( """Render a template-backed control input into a concrete control definition.""" template = template_input.template definition_template = template.definition_template - if isinstance(definition_template, dict): - for forbidden_key in ("enabled", "name"): - if forbidden_key in definition_template: - raise _render_error( - detail=f"Templates must not define top-level '{forbidden_key}'", - field=forbidden_key, - code="forbidden_template_field", - message=( - f"Templates must not define top-level '{forbidden_key}'. " - "Manage it outside the template." - ), - ) - if "condition" not in definition_template and ( - "selector" in definition_template or "evaluator" in definition_template - ): + if not isinstance(definition_template, dict): + raise _render_error( + detail="Templates must define a top-level control object", + field="template.definition_template", + code="invalid_definition_template_type", + message="definition_template must be a JSON object representing a control definition.", + ) + + for forbidden_key in ("enabled", "name"): + if forbidden_key in definition_template: raise _render_error( - detail="Templates must use the canonical 'condition' format", - field="condition", - code="legacy_condition_format_not_supported", + detail=f"Templates must not define top-level '{forbidden_key}'", + field=forbidden_key, + code="forbidden_template_field", message=( - "Templates must use the canonical 'condition' wrapper instead of " - "top-level selector/evaluator fields." + f"Templates must not define top-level '{forbidden_key}'. " + "Manage it outside the template." ), ) + if "condition" not in definition_template and ( + "selector" in definition_template or "evaluator" in definition_template + ): + raise _render_error( + detail="Templates must use the canonical 'condition' format", + field="condition", + code="legacy_condition_format_not_supported", + message=( + "Templates must use the canonical 'condition' wrapper instead of " + "top-level selector/evaluator fields." + ), + ) resolved_values = _resolve_template_values(template, template_input.template_values) reverse_path_map: dict[str, str] = {} diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index b726d977..9eed0f74 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -2,11 +2,15 @@ from __future__ import annotations +import json import uuid from copy import deepcopy from agent_control_models import EvaluationRequest, Step from fastapi.testclient import TestClient +from sqlalchemy import text + +from .conftest import engine def _template_payload() -> dict[str, object]: @@ -326,6 +330,7 @@ def test_create_template_backed_control_persists_template_metadata(client: TestC "step_name": "templated-step", } assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" + assert data["scope"]["step_names"] == ["templated-step"] def test_get_control_returns_template_metadata_for_template_backed_control( @@ -721,6 +726,98 @@ def test_template_update_preserves_enabled_value(client: TestClient) -> None: assert data["scope"]["step_names"] == ["updated-step"] +def test_template_update_accepts_different_template_structure(client: TestClient) -> None: + control_id, _ = _create_template_control_with_name(client) + agent_name = _assign_control_to_agent(client, control_id) + + old_behavior_response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert old_behavior_response.status_code == 200, old_behavior_response.text + assert old_behavior_response.json()["is_safe"] is False + + replacement_payload = _defaults_only_template_payload() + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": replacement_payload}, + ) + assert put_response.status_code == 200, put_response.text + + get_response = client.get(f"/api/v1/controls/{control_id}/data") + assert get_response.status_code == 200, get_response.text + data = get_response.json()["data"] + assert data["template"]["description"] == "List evaluator template" + assert data["template_values"] == { + "values": ["secret", "blocked"], + "logic": "any", + "case_sensitive": False, + } + assert data["condition"]["evaluator"]["name"] == "list" + assert data["condition"]["evaluator"]["config"] == { + "values": ["secret", "blocked"], + "logic": "any", + "case_sensitive": False, + } + assert data["scope"]["step_types"] == ["llm"] + + old_match_response = _evaluate_step( + client, + agent_name, + step_name="templated-step", + input_value="hello", + ) + assert old_match_response.status_code == 200, old_match_response.text + assert old_match_response.json()["is_safe"] is True + + new_match_response = _evaluate_step( + client, + agent_name, + step_name="other-step", + input_value="secret", + ) + assert new_match_response.status_code == 200, new_match_response.text + assert new_match_response.json()["is_safe"] is False + + +def test_template_update_defaults_enabled_to_true_when_stored_key_is_missing( + client: TestClient, +) -> None: + control_id = _create_template_control(client) + get_response = client.get(f"/api/v1/controls/{control_id}/data") + assert get_response.status_code == 200, get_response.text + stored_data = get_response.json()["data"] + stored_data.pop("enabled", None) + + with engine.begin() as conn: + conn.execute( + text("UPDATE controls SET data = CAST(:data AS JSONB) WHERE id = :id"), + {"data": json.dumps(stored_data), "id": control_id}, + ) + + updated_payload = _template_payload() + updated_payload["template_values"] = { + "pattern": "updated", + "step_name": "updated-step", + } + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": updated_payload}, + ) + assert put_response.status_code == 200, put_response.text + + refreshed_response = client.get(f"/api/v1/controls/{control_id}/data") + assert refreshed_response.status_code == 200, refreshed_response.text + data = refreshed_response.json()["data"] + assert data["enabled"] is True + assert data["template_values"] == { + "pattern": "updated", + "step_name": "updated-step", + } + + def test_template_validate_maps_missing_parameter_error(client: TestClient) -> None: payload = _template_payload() payload["template_values"] = {} @@ -782,6 +879,25 @@ def test_render_control_template_rejects_undefined_param_reference(client: TestC ) +def test_render_control_template_rejects_non_object_definition_template( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["definition_template"] = "not-an-object" # type: ignore[index] + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("field") == "template.definition_template" + and err.get("code") == "invalid_definition_template_type" + for err in body.get("errors", []) + ) + + def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> None: control_id = _create_template_control(client) raw_payload = deepcopy( @@ -988,6 +1104,34 @@ def test_render_control_template_rejects_agent_scoped_evaluator(client: TestClie ) +def test_render_control_template_remaps_param_bound_agent_scoped_evaluator_error( + client: TestClient, +) -> None: + payload = _template_payload() + payload["template"] = deepcopy(payload["template"]) + payload["template"]["parameters"]["evaluator_name"] = { # type: ignore[index] + "type": "string", + "label": "Evaluator Name", + } + payload["template"]["definition_template"]["condition"]["evaluator"]["name"] = { # type: ignore[index] + "$param": "evaluator_name", + } + payload["template_values"]["evaluator_name"] = "agent-x:custom" # type: ignore[index] + + response = client.post("/api/v1/control-templates/render", json=payload) + + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" + assert any( + err.get("field") == "template_values.evaluator_name" + and err.get("parameter") == "evaluator_name" + and err.get("rendered_field") == "condition.evaluator.name" + and err.get("code") == "template_parameter_invalid" + for err in body.get("errors", []) + ) + + def test_render_control_template_rejects_forbidden_top_level_template_fields( client: TestClient, ) -> None: From 90906d017807f31d0a45b0b44fe052801c337544 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:29:36 -0700 Subject: [PATCH 12/25] fix: satisfy lint in control input parser --- models/src/agent_control_models/server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/src/agent_control_models/server.py b/models/src/agent_control_models/server.py index 89c92ec1..c0967647 100644 --- a/models/src/agent_control_models/server.py +++ b/models/src/agent_control_models/server.py @@ -17,7 +17,9 @@ def _strip_slug_name(v: str) -> str: _CONTROL_DEFINITION_ADAPTER = TypeAdapter(ControlDefinition) _TEMPLATE_CONTROL_INPUT_ADAPTER = TypeAdapter(TemplateControlInput) _TEMPLATE_ONLY_CONTROL_FIELDS = frozenset({"template", "template_values"}) -_RAW_CONTROL_INPUT_FIELDS = frozenset(ControlDefinition.model_fields) - _TEMPLATE_ONLY_CONTROL_FIELDS +_RAW_CONTROL_INPUT_FIELDS = ( + frozenset(ControlDefinition.model_fields) - _TEMPLATE_ONLY_CONTROL_FIELDS +) _RAW_CONTROL_INPUT_FIELDS = _RAW_CONTROL_INPUT_FIELDS.union( { # Legacy flat leaf fields still accepted for raw controls. From 97f20c918ba344dc63236d3e1476e65cb3a2c533 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:38:22 -0700 Subject: [PATCH 13/25] test: add given-when-then comments to template coverage --- models/tests/test_control_templates.py | 30 +++++ sdks/python/tests/test_controls_api.py | 21 ++++ sdks/python/tests/test_local_evaluation.py | 8 ++ server/tests/test_control_templates.py | 129 +++++++++++++++++++++ 4 files changed, 188 insertions(+) diff --git a/models/tests/test_control_templates.py b/models/tests/test_control_templates.py index 517a7624..6217c2c0 100644 --- a/models/tests/test_control_templates.py +++ b/models/tests/test_control_templates.py @@ -44,10 +44,12 @@ def _nested_template_value(depth: int) -> object: def test_control_definition_requires_template_fields_together() -> None: + # Given: a rendered control that only includes template metadata without template values with pytest.raises( ValidationError, match="template and template_values must both be present or both absent", ): + # When: validating the control definition model ControlDefinition.model_validate( { "execution": "server", @@ -63,13 +65,16 @@ def test_control_definition_requires_template_fields_together() -> None: "template": VALID_TEMPLATE, } ) + # Then: the model rejects the partial template-backed control shape def test_control_definition_rejects_template_values_without_template() -> None: + # Given: a rendered control that only includes template values without template metadata with pytest.raises( ValidationError, match="template and template_values must both be present or both absent", ): + # When: validating the control definition model ControlDefinition.model_validate( { "execution": "server", @@ -85,13 +90,16 @@ def test_control_definition_rejects_template_values_without_template() -> None: "template_values": {"pattern": "hello"}, } ) + # Then: the model rejects the partial template-backed control shape def test_template_definition_rejects_invalid_parameter_name() -> None: + # Given: a template definition with an invalid parameter name with pytest.raises( ValidationError, match=r"Parameter names must match \[a-zA-Z_\]\[a-zA-Z0-9_\]\*", ): + # When: validating the template definition model TemplateDefinition.model_validate( { "parameters": { @@ -103,35 +111,43 @@ def test_template_definition_rejects_invalid_parameter_name() -> None: "definition_template": {}, } ) + # Then: the invalid parameter name is rejected def test_template_definition_rejects_excessive_nesting() -> None: + # Given: a template definition whose structure exceeds the nesting limit with pytest.raises( ValidationError, match="definition_template nesting depth exceeds maximum", ): + # When: validating the template definition model TemplateDefinition.model_validate( { "parameters": {}, "definition_template": _nested_template_value(12), } ) + # Then: the model rejects the deeply nested template def test_template_definition_rejects_excessive_size() -> None: + # Given: a template definition whose structure exceeds the size limit with pytest.raises( ValidationError, match="definition_template size exceeds maximum", ): + # When: validating the template definition model TemplateDefinition.model_validate( { "parameters": {}, "definition_template": list(range(1001)), } ) + # Then: the model rejects the oversized template def test_create_control_request_parses_template_payload_as_template_input() -> None: + # Given: a create request payload containing template-backed control input request = CreateControlRequest.model_validate( { "name": "template-control", @@ -142,10 +158,13 @@ def test_create_control_request_parses_template_payload_as_template_input() -> N } ) + # When: the request model parses the payload + # Then: the control input is discriminated as template-backed input assert isinstance(request.data, TemplateControlInput) def test_create_control_request_rejects_mixed_raw_and_template_payload() -> None: + # Given: a create request payload that mixes template and rendered control fields with pytest.raises( ValidationError, match=( @@ -153,6 +172,7 @@ def test_create_control_request_rejects_mixed_raw_and_template_payload() -> None "control fields" ), ): + # When: the request model parses the mixed payload CreateControlRequest.model_validate( { "name": "template-control", @@ -163,9 +183,11 @@ def test_create_control_request_rejects_mixed_raw_and_template_payload() -> None }, } ) + # Then: the mixed payload is rejected clearly def test_control_definition_can_round_trip_to_template_control_input() -> None: + # Given: a stored template-backed control definition control = ControlDefinition.model_validate( { "execution": "server", @@ -183,8 +205,10 @@ def test_control_definition_can_round_trip_to_template_control_input() -> None: } ) + # When: converting the stored control back into template input template_input = control.to_template_control_input() + # Then: the template metadata and values round-trip unchanged assert template_input.template.parameters["pattern"].label == "Pattern" assert template_input.template.parameters["pattern"].type == "regex_re2" assert template_input.template.definition_template == VALID_TEMPLATE["definition_template"] @@ -192,6 +216,7 @@ def test_control_definition_can_round_trip_to_template_control_input() -> None: def test_control_definition_to_template_control_input_rejects_raw_control() -> None: + # Given: a raw control definition without template metadata control = ControlDefinition.model_validate( { "execution": "server", @@ -207,11 +232,14 @@ def test_control_definition_to_template_control_input_rejects_raw_control() -> N } ) + # When: converting the raw control into template input with pytest.raises(ValueError, match="not template-backed"): control.to_template_control_input() + # Then: the helper rejects the non-template-backed control def test_control_definition_runtime_ignores_template_metadata() -> None: + # Given: a stored template-backed control definition with template metadata runtime_control = ControlDefinitionRuntime.model_validate( { "execution": "server", @@ -229,5 +257,7 @@ def test_control_definition_runtime_ignores_template_metadata() -> None: } ) + # When: validating it through the runtime-only model + # Then: runtime parsing succeeds while ignoring template metadata assert runtime_control.execution == "server" assert runtime_control.action.decision == "deny" diff --git a/sdks/python/tests/test_controls_api.py b/sdks/python/tests/test_controls_api.py index d822b01e..29aa8c02 100644 --- a/sdks/python/tests/test_controls_api.py +++ b/sdks/python/tests/test_controls_api.py @@ -13,13 +13,16 @@ @pytest.mark.asyncio async def test_list_controls_passes_template_backed_filter() -> None: + # Given: an SDK client stub and a template-backed list filter response = Mock() response.raise_for_status = Mock() response.json = Mock(return_value={"controls": [], "pagination": {}}) client = SimpleNamespace(http_client=SimpleNamespace(get=AsyncMock(return_value=response))) + # When: listing controls through the SDK wrapper await agent_control.controls.list_controls(client, template_backed=True) + # Then: the filter is forwarded to the API request client.http_client.get.assert_awaited_once_with( "/api/v1/controls", params={"limit": 20, "template_backed": True}, @@ -28,6 +31,7 @@ async def test_list_controls_passes_template_backed_filter() -> None: @pytest.mark.asyncio async def test_create_control_accepts_template_control_input() -> None: + # Given: an SDK client stub and template-backed control input response = Mock() response.raise_for_status = Mock() response.json = Mock(return_value={"control_id": 123}) @@ -58,8 +62,10 @@ async def test_create_control_accepts_template_control_input() -> None: } ) + # When: creating a control through the SDK wrapper await agent_control.controls.create_control(client, "templated", template_input) + # Then: the template values are serialized into the request body client.http_client.put.assert_awaited_once() _, kwargs = client.http_client.put.await_args assert kwargs["json"]["data"]["template_values"]["pattern"] == "hello" @@ -67,11 +73,13 @@ async def test_create_control_accepts_template_control_input() -> None: @pytest.mark.asyncio async def test_render_control_template_calls_preview_endpoint() -> None: + # Given: an SDK client stub and template preview input response = Mock() response.raise_for_status = Mock() response.json = Mock(return_value={"control": {"execution": "server"}}) client = SimpleNamespace(http_client=SimpleNamespace(post=AsyncMock(return_value=response))) + # When: rendering a control template through the SDK wrapper await agent_control.controls.render_control_template( client, template={ @@ -89,6 +97,7 @@ async def test_render_control_template_calls_preview_endpoint() -> None: template_values={}, ) + # Then: the SDK calls the preview endpoint with the expected payload client.http_client.post.assert_awaited_once_with( "/api/v1/control-templates/render", json={ @@ -111,6 +120,7 @@ async def test_render_control_template_calls_preview_endpoint() -> None: @pytest.mark.asyncio async def test_validate_control_data_accepts_template_control_input() -> None: + # Given: an SDK client stub and template-backed control input response = Mock() response.raise_for_status = Mock() response.json = Mock(return_value={"success": True}) @@ -141,8 +151,10 @@ async def test_validate_control_data_accepts_template_control_input() -> None: } ) + # When: validating template-backed control input through the SDK wrapper await agent_control.controls.validate_control_data(client, template_input) + # Then: the template-backed payload is posted to the validate endpoint client.http_client.post.assert_awaited_once() _, kwargs = client.http_client.post.await_args assert kwargs["json"]["data"]["template_values"]["pattern"] == "hello" @@ -156,6 +168,7 @@ async def test_validate_control_data_accepts_template_control_input() -> None: @pytest.mark.asyncio async def test_set_control_data_accepts_template_control_input() -> None: + # Given: an SDK client stub and template-backed control input response = Mock() response.raise_for_status = Mock() response.json = Mock(return_value={"success": True}) @@ -186,14 +199,17 @@ async def test_set_control_data_accepts_template_control_input() -> None: } ) + # When: updating control data through the SDK wrapper await agent_control.controls.set_control_data(client, 123, template_input) + # Then: the template values are serialized into the request body client.http_client.put.assert_awaited_once() _, kwargs = client.http_client.put.await_args assert kwargs["json"]["data"]["template_values"]["pattern"] == "hello" def test_to_template_control_input_reshapes_stored_control_data() -> None: + # Given: stored template-backed control data returned by the API template_input = agent_control.controls.to_template_control_input( { "execution": "server", @@ -230,11 +246,15 @@ def test_to_template_control_input_reshapes_stored_control_data() -> None: } ) + # When: reshaping the stored data into template input + # Then: the result is template-backed input with the original values assert isinstance(template_input, TemplateControlInput) assert template_input.template_values == {"pattern": "hello"} def test_to_template_control_input_rejects_raw_control_data() -> None: + # Given: raw control data without template metadata + # When: reshaping it into template-backed control input with pytest.raises(ValueError, match="not template-backed"): agent_control.controls.to_template_control_input( { @@ -250,3 +270,4 @@ def test_to_template_control_input_rejects_raw_control_data() -> None: "action": {"decision": "deny"}, } ) + # Then: the helper rejects the raw control data diff --git a/sdks/python/tests/test_local_evaluation.py b/sdks/python/tests/test_local_evaluation.py index f124fef4..9b56814a 100644 --- a/sdks/python/tests/test_local_evaluation.py +++ b/sdks/python/tests/test_local_evaluation.py @@ -305,12 +305,14 @@ async def test_server_only_template_backed_controls_still_call_server( llm_payload, ): """Template metadata must not break routing for server-executed cached controls.""" + # Given: a cached server-executed control that includes template metadata controls = [ add_template_metadata( make_control_dict(1, "templated_server_ctrl", execution="server"), ), ] + # Given: a client stub that returns a safe server evaluation response client = MagicMock(spec=AgentControlClient) mock_response = MagicMock() mock_response.json.return_value = {"is_safe": True, "confidence": 1.0} @@ -318,6 +320,7 @@ async def test_server_only_template_backed_controls_still_call_server( client.http_client = AsyncMock() client.http_client.post = AsyncMock(return_value=mock_response) + # When: evaluating with local controls enabled result = await check_evaluation_with_local( client=client, agent_name=agent_name, @@ -326,6 +329,7 @@ async def test_server_only_template_backed_controls_still_call_server( controls=controls, ) + # Then: the SDK still routes evaluation to the server client.http_client.post.assert_called_once() assert result.is_safe is True @@ -512,15 +516,18 @@ async def test_template_backed_local_control_executes_locally( llm_payload, ): """Template metadata should be ignored by runtime parsing in local evaluation.""" + # Given: a cached SDK-executed control that includes template metadata controls = [ add_template_metadata( make_control_dict(1, "templated_local_ctrl", execution="sdk", pattern=r"test"), ), ] + # Given: a client stub that would allow server calls if routing were wrong client = MagicMock(spec=AgentControlClient) client.http_client = AsyncMock() client.http_client.post = AsyncMock() + # When: evaluating with local controls enabled result = await check_evaluation_with_local( client=client, agent_name=agent_name, @@ -529,6 +536,7 @@ async def test_template_backed_local_control_executes_locally( controls=controls, ) + # Then: local evaluation succeeds and no server call is made client.http_client.post.assert_not_called() assert result.is_safe is False assert result.matches is not None diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 9eed0f74..93c9a881 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -256,8 +256,11 @@ def _evaluate_step( def test_render_control_template_preview_returns_rendered_control(client: TestClient) -> None: + # Given: a valid template-backed control payload + # When: rendering a template preview through the public API response = client.post("/api/v1/control-templates/render", json=_template_payload()) + # Then: the rendered control includes resolved template metadata and values assert response.status_code == 200, response.text control = response.json()["control"] assert control["enabled"] is True @@ -273,8 +276,11 @@ def test_render_control_template_preview_returns_rendered_control(client: TestCl def test_render_control_template_preview_uses_defaults_when_values_omitted( client: TestClient, ) -> None: + # Given: a template whose defaults fully satisfy all parameters + # When: rendering a template preview without explicit template values response = client.post("/api/v1/control-templates/render", json=_defaults_only_template_payload()) + # Then: the rendered control uses the parameter defaults assert response.status_code == 200, response.text control = response.json()["control"] assert control["template_values"] == { @@ -293,11 +299,14 @@ def test_render_control_template_preview_uses_defaults_when_values_omitted( def test_render_control_template_preview_rejects_excessive_definition_nesting( client: TestClient, ) -> None: + # Given: a template definition that exceeds the nesting limit payload = _template_payload() payload["template"]["definition_template"] = _nested_template_value(12) + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API rejects the request with a clear validation error assert response.status_code == 422, response.text body = response.json() assert body["detail"].startswith("Request validation failed") @@ -307,11 +316,14 @@ def test_render_control_template_preview_rejects_excessive_definition_nesting( def test_render_control_template_preview_rejects_excessive_definition_size( client: TestClient, ) -> None: + # Given: a template definition that exceeds the size limit payload = _template_payload() payload["template"]["definition_template"] = list(range(1001)) + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API rejects the oversized template definition assert response.status_code == 422, response.text body = response.json() assert body["detail"].startswith("Request validation failed") @@ -319,9 +331,13 @@ def test_render_control_template_preview_rejects_excessive_definition_size( def test_create_template_backed_control_persists_template_metadata(client: TestClient) -> None: + # Given: a valid template-backed control created through the API control_id = _create_template_control(client) + # When: fetching the stored control data response = client.get(f"/api/v1/controls/{control_id}/data") + + # Then: both template metadata and rendered values are persisted assert response.status_code == 200, response.text data = response.json()["data"] assert data["template"]["description"] == "Regex denial template" @@ -336,10 +352,13 @@ def test_create_template_backed_control_persists_template_metadata(client: TestC def test_get_control_returns_template_metadata_for_template_backed_control( client: TestClient, ) -> None: + # Given: a template-backed control created through the API control_id, control_name = _create_template_control_with_name(client) + # When: fetching the full control resource response = client.get(f"/api/v1/controls/{control_id}") + # Then: the response includes template metadata alongside normal control metadata assert response.status_code == 200, response.text body = response.json() assert body["id"] == control_id @@ -354,14 +373,17 @@ def test_get_control_returns_template_metadata_for_template_backed_control( def test_create_template_backed_control_persists_resolved_defaults_when_values_omitted( client: TestClient, ) -> None: + # Given: a template-backed control created without explicit template values control_id, _ = _create_template_control_with_name( client, _defaults_only_template_payload(), name_prefix="defaulted-template-control", ) + # When: fetching the stored control data response = client.get(f"/api/v1/controls/{control_id}/data") + # Then: the persisted template values and rendered config include resolved defaults assert response.status_code == 200, response.text data = response.json()["data"] assert data["template_values"] == { @@ -377,9 +399,11 @@ def test_create_template_backed_control_persists_resolved_defaults_when_values_o def test_template_backed_control_evaluates_after_policy_attachment(client: TestClient) -> None: + # Given: a template-backed control attached to an agent through a policy control_id, control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) + # When: evaluating a non-matching step and then a matching step safe_response = _evaluate_step( client, agent_name, @@ -395,6 +419,7 @@ def test_template_backed_control_evaluates_after_policy_attachment(client: TestC step_name="templated-step", input_value="hello", ) + # Then: only the matching step is denied by the attached template-backed control assert deny_response.status_code == 200, deny_response.text body = deny_response.json() assert body["is_safe"] is False @@ -404,9 +429,11 @@ def test_template_backed_control_evaluates_after_policy_attachment(client: TestC def test_template_backed_control_can_be_disabled_and_reenabled_in_evaluation( client: TestClient, ) -> None: + # Given: an attached template-backed control control_id, control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) + # When: evaluating before disabling, after disabling, and after re-enabling the control initial_response = _evaluate_step( client, agent_name, @@ -437,6 +464,7 @@ def test_template_backed_control_can_be_disabled_and_reenabled_in_evaluation( step_name="templated-step", input_value="hello", ) + # Then: evaluation reflects the user-managed enabled state assert reenabled_eval.status_code == 200, reenabled_eval.text body = reenabled_eval.json() assert body["is_safe"] is False @@ -444,10 +472,12 @@ def test_template_backed_control_can_be_disabled_and_reenabled_in_evaluation( def test_template_backed_control_rename_is_reflected_in_evaluation(client: TestClient) -> None: + # Given: an attached template-backed control control_id, _ = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) renamed_control_name = f"renamed-template-control-{uuid.uuid4()}" + # When: renaming the control and evaluating a matching step patch_response = client.patch( f"/api/v1/controls/{control_id}", json={"name": renamed_control_name}, @@ -462,6 +492,7 @@ def test_template_backed_control_rename_is_reflected_in_evaluation(client: TestC input_value="hello", ) + # Then: the evaluation match reflects the updated control name assert eval_response.status_code == 200, eval_response.text body = eval_response.json() assert body["is_safe"] is False @@ -471,14 +502,17 @@ def test_template_backed_control_rename_is_reflected_in_evaluation(client: TestC def test_template_backed_control_patch_updates_name_and_enabled_together( client: TestClient, ) -> None: + # Given: a template-backed control control_id, _ = _create_template_control_with_name(client) renamed_control_name = f"patched-template-control-{uuid.uuid4()}" + # When: patching both user-managed metadata fields together patch_response = client.patch( f"/api/v1/controls/{control_id}", json={"name": renamed_control_name, "enabled": False}, ) + # Then: the metadata updates persist without losing template metadata assert patch_response.status_code == 200, patch_response.text body = patch_response.json() assert body["name"] == renamed_control_name @@ -497,9 +531,11 @@ def test_template_backed_control_patch_updates_name_and_enabled_together( def test_template_backed_control_update_changes_scope_behavior(client: TestClient) -> None: + # Given: an attached template-backed control control_id, control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) + # When: updating the template values to change the rendered scope initial_eval = _evaluate_step( client, agent_name, @@ -535,6 +571,7 @@ def test_template_backed_control_update_changes_scope_behavior(client: TestClien step_name="updated-step", input_value="hello", ) + # Then: the old scope stops matching and the new scope starts matching assert updated_scope_eval.status_code == 200, updated_scope_eval.text body = updated_scope_eval.json() assert body["is_safe"] is False @@ -542,9 +579,11 @@ def test_template_backed_control_update_changes_scope_behavior(client: TestClien def test_template_backed_control_supports_direct_agent_attachment(client: TestClient) -> None: + # Given: a template-backed control attached directly to an agent control_id, control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id, via_policy=False) + # When: evaluating a matching step response = _evaluate_step( client, agent_name, @@ -552,6 +591,7 @@ def test_template_backed_control_supports_direct_agent_attachment(client: TestCl input_value="hello", ) + # Then: the directly attached control participates in evaluation assert response.status_code == 200, response.text body = response.json() assert body["is_safe"] is False @@ -561,6 +601,7 @@ def test_template_backed_control_supports_direct_agent_attachment(client: TestCl def test_template_backed_warn_control_evaluates_as_safe_with_warn_match( client: TestClient, ) -> None: + # Given: a template-backed control whose rendered action is warn payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"]["action"]["decision"] = "warn" # type: ignore[index] @@ -571,6 +612,7 @@ def test_template_backed_warn_control_evaluates_as_safe_with_warn_match( ) agent_name = _assign_control_to_agent(client, control_id) + # When: evaluating a matching step response = _evaluate_step( client, agent_name, @@ -578,6 +620,7 @@ def test_template_backed_warn_control_evaluates_as_safe_with_warn_match( input_value="hello", ) + # Then: the evaluation remains safe while returning a warn match assert response.status_code == 200, response.text body = response.json() assert body["is_safe"] is True @@ -589,6 +632,7 @@ def test_template_backed_warn_control_evaluates_as_safe_with_warn_match( def test_template_backed_control_preserves_falsey_values_and_uses_them_in_behavior( client: TestClient, ) -> None: + # Given: a template-backed control created with falsey template values payload = _case_sensitive_template_payload(values=[], case_sensitive=False) control_id, control_name = _create_template_control_with_name( client, @@ -597,6 +641,7 @@ def test_template_backed_control_preserves_falsey_values_and_uses_them_in_behavi ) agent_name = _assign_control_to_agent(client, control_id) + # When: inspecting stored data, evaluating, then updating to non-empty values get_response = client.get(f"/api/v1/controls/{control_id}/data") assert get_response.status_code == 200, get_response.text data = get_response.json()["data"] @@ -630,6 +675,7 @@ def test_template_backed_control_preserves_falsey_values_and_uses_them_in_behavi step_name="templated-step", input_value="hello", ) + # Then: falsey values persist and later updates change evaluation behavior assert deny_eval.status_code == 200, deny_eval.text body = deny_eval.json() assert body["is_safe"] is False @@ -639,6 +685,7 @@ def test_template_backed_control_preserves_falsey_values_and_uses_them_in_behavi def test_mixed_raw_and_template_backed_controls_obey_deny_precedence( client: TestClient, ) -> None: + # Given: an agent with both a template-backed deny control and a raw warn control template_control_id, template_control_name = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, template_control_id) @@ -660,6 +707,7 @@ def test_mixed_raw_and_template_backed_controls_obey_deny_precedence( add_control_response = client.post(f"/api/v1/policies/{policy_id}/controls/{raw_warn_control_id}") assert add_control_response.status_code == 200, add_control_response.text + # When: evaluating a step that matches both controls response = _evaluate_step( client, agent_name, @@ -667,6 +715,7 @@ def test_mixed_raw_and_template_backed_controls_obey_deny_precedence( input_value="hello", ) + # Then: both matches are returned and deny precedence makes the result unsafe assert response.status_code == 200, response.text body = response.json() assert body["is_safe"] is False @@ -678,14 +727,17 @@ def test_mixed_raw_and_template_backed_controls_obey_deny_precedence( def test_raw_control_can_be_replaced_with_template_backed_control(client: TestClient) -> None: + # Given: an existing raw control control_id = _create_raw_control(client) + # When: replacing its data with template-backed control input put_response = client.put( f"/api/v1/controls/{control_id}/data", json={"data": _template_payload()}, ) assert put_response.status_code == 200, put_response.text + # Then: the stored control becomes template-backed and persists rendered values get_response = client.get(f"/api/v1/controls/{control_id}/data") assert get_response.status_code == 200, get_response.text data = get_response.json()["data"] @@ -695,6 +747,7 @@ def test_raw_control_can_be_replaced_with_template_backed_control(client: TestCl def test_template_update_preserves_enabled_value(client: TestClient) -> None: + # Given: a template-backed control that has been disabled through the metadata patch API control_id = _create_template_control(client) patch_response = client.patch( @@ -703,6 +756,7 @@ def test_template_update_preserves_enabled_value(client: TestClient) -> None: ) assert patch_response.status_code == 200, patch_response.text + # When: updating the template-backed control data updated_payload = _template_payload() updated_payload["template_values"] = { "pattern": "updated", @@ -714,6 +768,7 @@ def test_template_update_preserves_enabled_value(client: TestClient) -> None: ) assert put_response.status_code == 200, put_response.text + # Then: the stored enabled value is preserved across the template update get_response = client.get(f"/api/v1/controls/{control_id}/data") assert get_response.status_code == 200, get_response.text data = get_response.json()["data"] @@ -727,9 +782,11 @@ def test_template_update_preserves_enabled_value(client: TestClient) -> None: def test_template_update_accepts_different_template_structure(client: TestClient) -> None: + # Given: an attached template-backed control using the regex template shape control_id, _ = _create_template_control_with_name(client) agent_name = _assign_control_to_agent(client, control_id) + # When: replacing its data with a different template structure old_behavior_response = _evaluate_step( client, agent_name, @@ -778,6 +835,7 @@ def test_template_update_accepts_different_template_structure(client: TestClient step_name="other-step", input_value="secret", ) + # Then: the stored template metadata and runtime behavior both follow the new template assert new_match_response.status_code == 200, new_match_response.text assert new_match_response.json()["is_safe"] is False @@ -785,6 +843,7 @@ def test_template_update_accepts_different_template_structure(client: TestClient def test_template_update_defaults_enabled_to_true_when_stored_key_is_missing( client: TestClient, ) -> None: + # Given: a stored template-backed control whose persisted JSON is missing enabled control_id = _create_template_control(client) get_response = client.get(f"/api/v1/controls/{control_id}/data") assert get_response.status_code == 200, get_response.text @@ -797,6 +856,7 @@ def test_template_update_defaults_enabled_to_true_when_stored_key_is_missing( {"data": json.dumps(stored_data), "id": control_id}, ) + # When: updating the template-backed control data updated_payload = _template_payload() updated_payload["template_values"] = { "pattern": "updated", @@ -808,6 +868,7 @@ def test_template_update_defaults_enabled_to_true_when_stored_key_is_missing( ) assert put_response.status_code == 200, put_response.text + # Then: the update path preserves the default enabled=True behavior refreshed_response = client.get(f"/api/v1/controls/{control_id}/data") assert refreshed_response.status_code == 200, refreshed_response.text data = refreshed_response.json()["data"] @@ -819,11 +880,14 @@ def test_template_update_defaults_enabled_to_true_when_stored_key_is_missing( def test_template_validate_maps_missing_parameter_error(client: TestClient) -> None: + # Given: a template-backed control payload missing a required parameter value payload = _template_payload() payload["template_values"] = {} + # When: validating the payload through the public API response = client.post("/api/v1/controls/validate", json={"data": payload}) + # Then: the error is remapped back to the missing template parameter assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" @@ -835,21 +899,27 @@ def test_template_validate_maps_missing_parameter_error(client: TestClient) -> N def test_template_validate_succeeds_with_defaults_only_payload(client: TestClient) -> None: + # Given: a template-backed control payload whose defaults satisfy every parameter + # When: validating it through the public API response = client.post( "/api/v1/controls/validate", json={"data": _defaults_only_template_payload()}, ) + # Then: validation succeeds without requiring explicit template values assert response.status_code == 200, response.text assert response.json()["success"] is True def test_render_control_template_rejects_unknown_template_value_key(client: TestClient) -> None: + # Given: a template payload with an undeclared template value key payload = _template_payload() payload["template_values"] = {"pattern": "hello", "unknown": "value"} + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API rejects the unknown parameter key clearly assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" @@ -861,14 +931,17 @@ def test_render_control_template_rejects_unknown_template_value_key(client: Test def test_render_control_template_rejects_undefined_param_reference(client: TestClient) -> None: + # Given: a template definition that references an undeclared parameter payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["pattern"] = { # type: ignore[index] "$param": "undefined_pattern", } + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API reports the undefined parameter reference on the rendered field assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -882,12 +955,15 @@ def test_render_control_template_rejects_undefined_param_reference(client: TestC def test_render_control_template_rejects_non_object_definition_template( client: TestClient, ) -> None: + # Given: a template whose top-level definition_template is not an object payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"] = "not-an-object" # type: ignore[index] + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API rejects the template with a clear top-level type error assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -899,6 +975,7 @@ def test_render_control_template_rejects_non_object_definition_template( def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> None: + # Given: a template-backed control and a raw replacement payload control_id = _create_template_control(client) raw_payload = deepcopy( { @@ -917,11 +994,13 @@ def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> N } ) + # When: replacing template-backed data with raw control data response = client.put( f"/api/v1/controls/{control_id}/data", json={"data": raw_payload}, ) + # Then: the API rejects the conversion back to raw control data assert response.status_code == 409 assert response.json()["error_code"] == "CONTROL_TEMPLATE_CONFLICT" @@ -929,9 +1008,11 @@ def test_template_backed_control_rejects_raw_put_update(client: TestClient) -> N def test_create_control_rejects_mixed_raw_and_template_payload_at_api_boundary( client: TestClient, ) -> None: + # Given: a create payload that mixes template-backed and rendered control fields payload = _template_payload() payload["execution"] = "server" + # When: creating the control through the public API response = client.put( "/api/v1/controls", json={ @@ -940,6 +1021,7 @@ def test_create_control_rejects_mixed_raw_and_template_payload_at_api_boundary( }, ) + # Then: the request is rejected with the mixed-payload validation message assert response.status_code == 422 body = response.json() assert body["error_code"] == "VALIDATION_ERROR" @@ -952,11 +1034,14 @@ def test_create_control_rejects_mixed_raw_and_template_payload_at_api_boundary( def test_validate_control_rejects_mixed_raw_and_template_payload_at_api_boundary( client: TestClient, ) -> None: + # Given: a validate payload that mixes template-backed and rendered control fields payload = _template_payload() payload["execution"] = "server" + # When: validating the mixed payload through the public API response = client.post("/api/v1/controls/validate", json={"data": payload}) + # Then: the request is rejected with the mixed-payload validation message assert response.status_code == 422 body = response.json() assert body["error_code"] == "VALIDATION_ERROR" @@ -967,9 +1052,13 @@ def test_validate_control_rejects_mixed_raw_and_template_payload_at_api_boundary def test_list_controls_includes_template_backed_flag_and_filter(client: TestClient) -> None: + # Given: a template-backed control in the system control_id = _create_template_control(client) + # When: listing controls with the template_backed=true filter response = client.get("/api/v1/controls", params={"template_backed": True}) + + # Then: the matching summary row is marked as template-backed assert response.status_code == 200, response.text controls = response.json()["controls"] template_control = next(control for control in controls if control["id"] == control_id) @@ -977,10 +1066,14 @@ def test_list_controls_includes_template_backed_flag_and_filter(client: TestClie def test_list_controls_template_backed_false_returns_only_raw_controls(client: TestClient) -> None: + # Given: one template-backed control and one raw control template_control_id = _create_template_control(client) raw_control_id = _create_raw_control(client) + # When: listing controls with the template_backed=false filter response = client.get("/api/v1/controls", params={"template_backed": False}) + + # Then: only raw controls are returned assert response.status_code == 200, response.text control_ids = {control["id"] for control in response.json()["controls"]} assert raw_control_id in control_ids @@ -988,21 +1081,27 @@ def test_list_controls_template_backed_false_returns_only_raw_controls(client: T def test_render_control_template_rejects_extra_request_fields(client: TestClient) -> None: + # Given: a render request payload with extra top-level fields payload = _template_payload() payload["execution"] = "server" + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: request validation rejects the extra fields assert response.status_code == 422 assert response.json()["error_code"] == "VALIDATION_ERROR" def test_render_control_template_maps_invalid_regex_parameter(client: TestClient) -> None: + # Given: a template payload with an invalid regex parameter value payload = _template_payload() payload["template_values"] = {"pattern": "["} + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the regex validation error is remapped back to the template parameter assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" @@ -1016,6 +1115,7 @@ def test_render_control_template_maps_invalid_regex_parameter(client: TestClient def test_render_control_template_rejects_optional_referenced_parameter_without_default( client: TestClient, ) -> None: + # Given: an optional parameter that is referenced in the template without a default payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["parameters"]["step_name"] = { # type: ignore[index] @@ -1024,8 +1124,10 @@ def test_render_control_template_rejects_optional_referenced_parameter_without_d "required": False, } + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API reports that referenced optional parameters require defaults assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -1037,6 +1139,7 @@ def test_render_control_template_rejects_optional_referenced_parameter_without_d def test_render_control_template_rejects_malformed_param_binding(client: TestClient) -> None: + # Given: a malformed $param binding object with extra keys payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["pattern"] = { # type: ignore[index] @@ -1044,8 +1147,10 @@ def test_render_control_template_rejects_malformed_param_binding(client: TestCli "extra": True, } + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the invalid binding is rejected as a template render error assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -1053,14 +1158,17 @@ def test_render_control_template_rejects_malformed_param_binding(client: TestCli def test_render_control_template_rejects_non_string_param_reference(client: TestClient) -> None: + # Given: a malformed $param binding whose reference is not a string payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["pattern"] = { # type: ignore[index] "$param": 123, } + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the invalid binding is rejected as a template render error assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -1068,6 +1176,7 @@ def test_render_control_template_rejects_non_string_param_reference(client: Test def test_render_control_template_rejects_unused_parameter(client: TestClient) -> None: + # Given: a template definition that declares an unused parameter payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["parameters"]["unused"] = { # type: ignore[index] @@ -1076,8 +1185,10 @@ def test_render_control_template_rejects_unused_parameter(client: TestClient) -> "default": "still-unused", } + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the unused parameter is reported on the template definition assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -1089,12 +1200,15 @@ def test_render_control_template_rejects_unused_parameter(client: TestClient) -> def test_render_control_template_rejects_agent_scoped_evaluator(client: TestClient) -> None: + # Given: a template definition that uses an agent-scoped evaluator directly payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"]["condition"]["evaluator"]["name"] = "agent-x:custom" # type: ignore[index] + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API rejects agent-scoped evaluators for template-backed controls assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -1107,6 +1221,7 @@ def test_render_control_template_rejects_agent_scoped_evaluator(client: TestClie def test_render_control_template_remaps_param_bound_agent_scoped_evaluator_error( client: TestClient, ) -> None: + # Given: a template whose evaluator name comes from a bound parameter payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["parameters"]["evaluator_name"] = { # type: ignore[index] @@ -1118,8 +1233,10 @@ def test_render_control_template_remaps_param_bound_agent_scoped_evaluator_error } payload["template_values"]["evaluator_name"] = "agent-x:custom" # type: ignore[index] + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the agent-scoped evaluator error is remapped to the bound parameter assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" @@ -1135,13 +1252,16 @@ def test_render_control_template_remaps_param_bound_agent_scoped_evaluator_error def test_render_control_template_rejects_forbidden_top_level_template_fields( client: TestClient, ) -> None: + # Given: templates that try to manage forbidden top-level control fields for forbidden_field, value in (("enabled", True), ("name", "templated-name")): payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"][forbidden_field] = value # type: ignore[index] + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the forbidden field is rejected clearly assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -1152,6 +1272,7 @@ def test_render_control_template_rejects_forbidden_top_level_template_fields( def test_render_control_template_rejects_legacy_flat_format(client: TestClient) -> None: + # Given: a template that uses the legacy flat selector/evaluator format payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"] = { # type: ignore[index] @@ -1165,8 +1286,10 @@ def test_render_control_template_rejects_legacy_flat_format(client: TestClient) "action": {"decision": "deny"}, } + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the API requires the canonical condition wrapper assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" @@ -1179,6 +1302,7 @@ def test_render_control_template_rejects_legacy_flat_format(client: TestClient) def test_render_control_template_rejects_invalid_parameter_name_at_api_boundary( client: TestClient, ) -> None: + # Given: a render request with an invalid template parameter name payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["parameters"] = { # type: ignore[index] @@ -1188,8 +1312,10 @@ def test_render_control_template_rejects_invalid_parameter_name_at_api_boundary( } } + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: request validation rejects the invalid parameter name assert response.status_code == 422 assert response.json()["error_code"] == "VALIDATION_ERROR" @@ -1197,12 +1323,15 @@ def test_render_control_template_rejects_invalid_parameter_name_at_api_boundary( def test_render_control_template_keeps_non_parameterized_errors_on_rendered_fields( client: TestClient, ) -> None: + # Given: a template whose rendered action is invalid independently of any parameter payload = _template_payload() payload["template"] = deepcopy(payload["template"]) payload["template"]["definition_template"]["action"]["decision"] = "block" # type: ignore[index] + # When: rendering a template preview response = client.post("/api/v1/control-templates/render", json=payload) + # Then: the error remains attached to the rendered field path assert response.status_code == 422 body = response.json() assert body["error_code"] == "TEMPLATE_RENDER_ERROR" From a06c83fc6c520e83dfb711ac6dae56c3b7713d3d Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Tue, 31 Mar 2026 23:49:39 -0700 Subject: [PATCH 14/25] test: add preview to persistence template flow coverage --- server/tests/test_control_templates.py | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 93c9a881..1f7c432a 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -781,6 +781,34 @@ def test_template_update_preserves_enabled_value(client: TestClient) -> None: assert data["scope"]["step_names"] == ["updated-step"] +def test_template_update_preview_matches_persisted_rendered_control(client: TestClient) -> None: + # Given: an existing template-backed control and updated template values + control_id = _create_template_control(client) + updated_payload = _template_payload() + updated_payload["template_values"] = { + "pattern": "updated", + "step_name": "updated-step", + } + + # When: previewing the updated template render and then persisting the update + preview_response = client.post("/api/v1/control-templates/render", json=updated_payload) + assert preview_response.status_code == 200, preview_response.text + preview_control = preview_response.json()["control"] + + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": updated_payload}, + ) + assert put_response.status_code == 200, put_response.text + + get_response = client.get(f"/api/v1/controls/{control_id}/data") + + # Then: the persisted rendered control matches the preview output + assert get_response.status_code == 200, get_response.text + persisted_control = get_response.json()["data"] + assert persisted_control == preview_control + + def test_template_update_accepts_different_template_structure(client: TestClient) -> None: # Given: an attached template-backed control using the regex template shape control_id, _ = _create_template_control_with_name(client) From ef13298538685948d7fc643f8b5a618c21bc189e Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 00:11:37 -0700 Subject: [PATCH 15/25] test: add negative template control failure coverage --- server/tests/test_control_templates.py | 88 ++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 1f7c432a..5ab8eb05 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -398,6 +398,37 @@ def test_create_template_backed_control_persists_resolved_defaults_when_values_o } +def test_create_template_backed_control_failure_does_not_persist_control( + client: TestClient, +) -> None: + # Given: an invalid template-backed create payload and the current control list + invalid_payload = _template_payload() + invalid_payload["template_values"] = {"pattern": "["} + control_name = f"invalid-template-control-{uuid.uuid4()}" + + before_response = client.get("/api/v1/controls") + assert before_response.status_code == 200, before_response.text + before_controls = before_response.json()["controls"] + before_ids = {control["id"] for control in before_controls} + + # When: creating the invalid template-backed control + response = client.put( + "/api/v1/controls", + json={"name": control_name, "data": invalid_payload}, + ) + + # Then: the request fails and no control is persisted + assert response.status_code == 422, response.text + assert response.json()["error_code"] == "TEMPLATE_PARAMETER_INVALID" + + after_response = client.get("/api/v1/controls") + assert after_response.status_code == 200, after_response.text + after_controls = after_response.json()["controls"] + after_ids = {control["id"] for control in after_controls} + assert after_ids == before_ids + assert all(control["name"] != control_name for control in after_controls) + + def test_template_backed_control_evaluates_after_policy_attachment(client: TestClient) -> None: # Given: a template-backed control attached to an agent through a policy control_id, control_name = _create_template_control_with_name(client) @@ -746,6 +777,36 @@ def test_raw_control_can_be_replaced_with_template_backed_control(client: TestCl assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" +def test_raw_control_failed_template_replacement_does_not_mutate_raw_control( + client: TestClient, +) -> None: + # Given: an existing raw control and invalid template-backed replacement input + control_id = _create_raw_control(client) + original_response = client.get(f"/api/v1/controls/{control_id}/data") + assert original_response.status_code == 200, original_response.text + original_data = original_response.json()["data"] + + invalid_payload = _template_payload() + invalid_payload["template_values"] = {"pattern": "["} + + # When: replacing the raw control with invalid template-backed input + response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": invalid_payload}, + ) + + # Then: the conversion fails and the raw control stays unchanged + assert response.status_code == 422, response.text + assert response.json()["error_code"] == "TEMPLATE_PARAMETER_INVALID" + + refreshed_response = client.get(f"/api/v1/controls/{control_id}/data") + assert refreshed_response.status_code == 200, refreshed_response.text + refreshed_data = refreshed_response.json()["data"] + assert refreshed_data == original_data + assert "template" not in refreshed_data + assert "template_values" not in refreshed_data + + def test_template_update_preserves_enabled_value(client: TestClient) -> None: # Given: a template-backed control that has been disabled through the metadata patch API control_id = _create_template_control(client) @@ -781,6 +842,33 @@ def test_template_update_preserves_enabled_value(client: TestClient) -> None: assert data["scope"]["step_names"] == ["updated-step"] +def test_template_update_failure_does_not_mutate_existing_template_backed_control( + client: TestClient, +) -> None: + # Given: an existing template-backed control and invalid updated template values + control_id = _create_template_control(client) + original_response = client.get(f"/api/v1/controls/{control_id}/data") + assert original_response.status_code == 200, original_response.text + original_data = original_response.json()["data"] + + invalid_payload = _template_payload() + invalid_payload["template_values"] = {"pattern": "["} + + # When: updating the stored template-backed control with invalid input + response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": invalid_payload}, + ) + + # Then: the update fails and the stored control remains unchanged + assert response.status_code == 422, response.text + assert response.json()["error_code"] == "TEMPLATE_PARAMETER_INVALID" + + refreshed_response = client.get(f"/api/v1/controls/{control_id}/data") + assert refreshed_response.status_code == 200, refreshed_response.text + assert refreshed_response.json()["data"] == original_data + + def test_template_update_preview_matches_persisted_rendered_control(client: TestClient) -> None: # Given: an existing template-backed control and updated template values control_id = _create_template_control(client) From b343fce38910986aa2893ce757559213c77e6c1f Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 00:19:59 -0700 Subject: [PATCH 16/25] refactor: simplify template control server flow --- .../endpoints/controls.py | 117 ++++++++++-------- .../services/control_templates.py | 100 +++++++++------ 2 files changed, 130 insertions(+), 87 deletions(-) diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index ce0ae9d9..30cd8efc 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -58,6 +58,8 @@ template_router = APIRouter(prefix="/control-templates", tags=["controls"]) _logger = get_logger(__name__) + + def _serialize_control_definition(control_def: ControlDefinition) -> dict[str, object]: """Serialize control data for storage while omitting null scope fields.""" data_json = control_def.model_dump( @@ -78,6 +80,36 @@ def _is_template_backed_payload(data: object) -> bool: return isinstance(data, dict) and data.get("template") is not None +def _enabled_from_stored_payload(data: object) -> bool: + """Return the persisted enabled flag, defaulting to True when absent.""" + if not isinstance(data, dict): + return True + raw_enabled = data.get("enabled", True) + return raw_enabled if type(raw_enabled) is bool else True + + +def _template_backed_raw_update_conflict(control_id: int) -> ConflictError: + """Return the v1 conflict raised when raw data updates target template-backed controls.""" + return ConflictError( + error_code=ErrorCode.CONTROL_TEMPLATE_CONFLICT, + detail="Template-backed controls cannot be updated with raw control data in v1", + resource="Control", + resource_id=str(control_id), + hint=( + "Submit template input to update this control, or delete and recreate " + "it as a raw control." + ), + errors=[ + ValidationErrorItem( + resource="Control", + field="data", + code="template_backed_control_conflict", + message="Template-backed controls must be updated with template input.", + ) + ], + ) + + async def _render_and_validate_template_input( template_input: TemplateControlInput, *, @@ -97,6 +129,33 @@ async def _render_and_validate_template_input( return rendered.control +async def _materialize_control_input( + control_input: ControlDefinition | TemplateControlInput, + *, + db: AsyncSession, + current_payload: object | None = None, + control_id: int | None = None, +) -> ControlDefinition: + """Resolve raw or template-backed input into a validated control definition.""" + if isinstance(control_input, TemplateControlInput): + enabled = ( + True if current_payload is None else _enabled_from_stored_payload(current_payload) + ) + return await _render_and_validate_template_input( + control_input, + db=db, + enabled=enabled, + ) + + if current_payload is not None and _is_template_backed_payload(current_payload): + if control_id is None: + raise RuntimeError("control_id is required for template-backed raw updates") + raise _template_backed_raw_update_conflict(control_id) + + await _validate_control_definition(control_input, db) + return control_input + + async def _validate_control_definition( control_def: ControlDefinition, db: AsyncSession ) -> None: @@ -321,15 +380,7 @@ async def create_control( hint="Choose a different name or update the existing control.", ) - if isinstance(request.data, TemplateControlInput): - control_def = await _render_and_validate_template_input( - request.data, - db=db, - enabled=True, - ) - else: - control_def = request.data - await _validate_control_definition(control_def, db) + control_def = await _materialize_control_input(request.data, db=db) control_data = _serialize_control_definition(control_def) control = Control(name=request.name, data=control_data) @@ -501,39 +552,12 @@ async def set_control_data( hint="Verify the control ID is correct and the control has been created.", ) - current_template_backed = _is_template_backed_payload(control.data) - if isinstance(request.data, TemplateControlInput): - current_enabled = True - if isinstance(control.data, dict): - raw_enabled = control.data.get("enabled", True) - current_enabled = raw_enabled if type(raw_enabled) is bool else True - control_def = await _render_and_validate_template_input( - request.data, - db=db, - enabled=current_enabled, - ) - else: - if current_template_backed: - raise ConflictError( - error_code=ErrorCode.CONTROL_TEMPLATE_CONFLICT, - detail="Template-backed controls cannot be updated with raw control data in v1", - resource="Control", - resource_id=str(control_id), - hint=( - "Submit template input to update this control, or delete and recreate " - "it as a raw control." - ), - errors=[ - ValidationErrorItem( - resource="Control", - field="data", - code="template_backed_control_conflict", - message="Template-backed controls must be updated with template input.", - ) - ], - ) - control_def = request.data - await _validate_control_definition(control_def, db) + control_def = await _materialize_control_input( + request.data, + db=db, + current_payload=control.data, + control_id=control_id, + ) control.data = _serialize_control_definition(control_def) try: @@ -571,14 +595,7 @@ async def validate_control_data( Returns: ValidateControlDataResponse with success=True if valid """ - if isinstance(request.data, TemplateControlInput): - await _render_and_validate_template_input( - request.data, - db=db, - enabled=True, - ) - else: - await _validate_control_definition(request.data, db) + await _materialize_control_input(request.data, db=db) return ValidateControlDataResponse(success=True) diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index af93b416..66c965ba 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -101,6 +101,41 @@ def _parameter_default( return _TEMPLATE_VALUE_MISSING if default is None else cast(TemplateValue, default) +def _parameter_invalid_type( + parameter_name: str, + parameter_definition: TemplateParameterDefinition, + *, + expected: str, + value: TemplateValue, +) -> APIValidationError: + """Create a standard invalid-type error for template parameter values.""" + return _parameter_error( + parameter_name, + parameter_definition, + f"Parameter '{parameter_definition.label}' must be {expected}.", + code="invalid_type", + value=value, + ) + + +def _require_string_parameter( + parameter_name: str, + parameter_definition: TemplateParameterDefinition, + value: TemplateValue, + *, + expected: str, +) -> str: + """Return a string parameter value or raise a parameter-focused type error.""" + if not isinstance(value, str): + raise _parameter_invalid_type( + parameter_name, + parameter_definition, + expected=expected, + value=value, + ) + return value + + def _coerce_parameter_value( parameter_name: str, parameter_definition: TemplateParameterDefinition, @@ -109,38 +144,32 @@ def _coerce_parameter_value( """Validate a concrete parameter value against its parameter definition.""" parameter_type = parameter_definition.type if parameter_type == "string": - if not isinstance(value, str): - raise _parameter_error( - parameter_name, - parameter_definition, - f"Parameter '{parameter_definition.label}' must be a string.", - code="invalid_type", - value=value, - ) - return value + return _require_string_parameter( + parameter_name, + parameter_definition, + value, + expected="a string", + ) if parameter_type == "string_list": if not isinstance(value, list) or any(not isinstance(item, str) for item in value): - raise _parameter_error( + raise _parameter_invalid_type( parameter_name, parameter_definition, - f"Parameter '{parameter_definition.label}' must be a list of strings.", - code="invalid_type", + expected="a list of strings", value=value, ) return list(value) if parameter_type == "enum": - if not isinstance(value, str): - raise _parameter_error( - parameter_name, - parameter_definition, - f"Parameter '{parameter_definition.label}' must be a string enum value.", - code="invalid_type", - value=value, - ) + enum_value = _require_string_parameter( + parameter_name, + parameter_definition, + value, + expected="a string enum value", + ) enum_definition = cast(EnumTemplateParameter, parameter_definition) - if value not in enum_definition.allowed_values: + if enum_value not in enum_definition.allowed_values: raise _parameter_error( parameter_name, parameter_definition, @@ -149,32 +178,29 @@ def _coerce_parameter_value( f"{enum_definition.allowed_values}." ), code="invalid_enum_value", - value=value, + value=enum_value, ) - return value + return enum_value if parameter_type == "boolean": if type(value) is not bool: - raise _parameter_error( + raise _parameter_invalid_type( parameter_name, parameter_definition, - f"Parameter '{parameter_definition.label}' must be a boolean.", - code="invalid_type", + expected="a boolean", value=value, ) return value if parameter_type == "regex_re2": - if not isinstance(value, str): - raise _parameter_error( - parameter_name, - parameter_definition, - f"Parameter '{parameter_definition.label}' must be a string regex pattern.", - code="invalid_type", - value=value, - ) + pattern = _require_string_parameter( + parameter_name, + parameter_definition, + value, + expected="a string regex pattern", + ) try: - re2.compile(value) + re2.compile(pattern) except re2.error as exc: raise _parameter_error( parameter_name, @@ -184,9 +210,9 @@ def _coerce_parameter_value( f"Invalid regex pattern: {exc}" ), code="invalid_regex", - value=value, + value=pattern, ) from exc - return value + return pattern raise _render_error( detail=f"Unsupported template parameter type '{parameter_type}'", From 058cb060f6571bb5700d9de65f9e497263f41907 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 14:03:31 -0700 Subject: [PATCH 17/25] feat: support unrendered template controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Template controls can now be created without providing parameter values. The server validates the template structure but skips rendering, storing the template metadata with enabled=false. Unrendered controls are visible in listings, attachable to agents, but excluded from evaluation. - Add UnrenderedTemplateControl model for GET response union - Create endpoint branches: complete values render, incomplete store unrendered - Update endpoint supports unrendered→rendered transition when values provided - GET endpoints discriminate via condition key presence in stored JSONB - PATCH rejects enabling unrendered templates with 422 - ControlSummary gains template_rendered field - Runtime evaluation query skips unrendered templates - Agent policy validation skips unrendered templates - 10 new server tests covering the full unrendered lifecycle --- models/src/agent_control_models/__init__.py | 2 + models/src/agent_control_models/controls.py | 21 ++ models/src/agent_control_models/policy.py | 9 +- models/src/agent_control_models/server.py | 29 ++- .../agent_control_server/endpoints/agents.py | 14 +- .../endpoints/controls.py | 130 +++++++++-- .../services/control_templates.py | 60 +++++ .../agent_control_server/services/controls.py | 20 ++ server/tests/test_control_templates.py | 220 ++++++++++++++++++ 9 files changed, 471 insertions(+), 34 deletions(-) diff --git a/models/src/agent_control_models/__init__.py b/models/src/agent_control_models/__init__.py index 29eab30a..179a5945 100644 --- a/models/src/agent_control_models/__init__.py +++ b/models/src/agent_control_models/__init__.py @@ -39,6 +39,7 @@ TemplateParameterBase, TemplateParameterDefinition, TemplateValue, + UnrenderedTemplateControl, ) from .errors import ( ERROR_TITLES, @@ -133,6 +134,7 @@ "TemplateParameterDefinition", "TemplateDefinition", "TemplateControlInput", + "UnrenderedTemplateControl", # Error models "ProblemDetail", "ErrorCode", diff --git a/models/src/agent_control_models/controls.py b/models/src/agent_control_models/controls.py index 65993fc7..f075dc4e 100644 --- a/models/src/agent_control_models/controls.py +++ b/models/src/agent_control_models/controls.py @@ -466,6 +466,27 @@ class TemplateControlInput(BaseModel): ) +class UnrenderedTemplateControl(BaseModel): + """Stored state of a template control that hasn't been rendered yet. + + An unrendered template has a template definition and possibly partial + parameter values, but no concrete condition/action/execution fields. + It is always ``enabled=False`` and excluded from evaluation. + """ + + template: TemplateDefinition = Field( + ..., description="Template definition awaiting parameter values" + ) + template_values: dict[str, TemplateValue] = Field( + default_factory=dict, + description="Partial or empty parameter values", + ) + enabled: Literal[False] = Field( + False, + description="Unrendered templates are always disabled", + ) + + class ControlAction(BaseModel): """What to do when control matches.""" diff --git a/models/src/agent_control_models/policy.py b/models/src/agent_control_models/policy.py index aafb895c..c6bb9b3e 100644 --- a/models/src/agent_control_models/policy.py +++ b/models/src/agent_control_models/policy.py @@ -1,17 +1,18 @@ from .base import BaseModel -from .controls import ControlDefinition +from .controls import ControlDefinition, UnrenderedTemplateControl class Control(BaseModel): """A control with identity and configuration. - Note: Only fully-configured controls (with valid ControlDefinition) - are returned from API endpoints. Unconfigured controls are filtered out. + For rendered controls (raw or template-backed), ``control`` is a + ``ControlDefinition``. For unrendered template controls, ``control`` + is an ``UnrenderedTemplateControl``. """ id: int name: str - control: ControlDefinition + control: ControlDefinition | UnrenderedTemplateControl class Policy(BaseModel): diff --git a/models/src/agent_control_models/server.py b/models/src/agent_control_models/server.py index c0967647..5a565acd 100644 --- a/models/src/agent_control_models/server.py +++ b/models/src/agent_control_models/server.py @@ -5,7 +5,13 @@ from .agent import Agent, StepSchema from .base import BaseModel -from .controls import ControlDefinition, TemplateControlInput, TemplateDefinition, TemplateValue +from .controls import ( + ControlDefinition, + TemplateControlInput, + TemplateDefinition, + TemplateValue, + UnrenderedTemplateControl, +) from .policy import Control @@ -302,8 +308,13 @@ class GetControlResponse(BaseModel): id: int = Field(..., description="Control ID") name: str = Field(..., description="Control name") - data: ControlDefinition | None = Field( - None, description="Control configuration data (None if not yet configured)" + data: ControlDefinition | UnrenderedTemplateControl | None = Field( + None, + description=( + "Control configuration data. A ControlDefinition for raw/rendered " + "controls, an UnrenderedTemplateControl for unrendered templates, " + "or None if not yet configured." + ), ) @@ -332,7 +343,9 @@ class RemoveAgentControlResponse(BaseModel): class GetControlDataResponse(BaseModel): - data: ControlDefinition = Field(description="Control data payload") + data: ControlDefinition | UnrenderedTemplateControl = Field( + description="Control data payload (rendered control or unrendered template)" + ) class SetControlDataRequest(BaseModel): @@ -469,6 +482,14 @@ class ControlSummary(BaseModel): False, description="Whether the control was created from a template", ) + template_rendered: bool | None = Field( + None, + description=( + "Whether a template-backed control has been rendered. " + "True for rendered templates, False for unrendered templates, " + "None for non-template controls." + ), + ) used_by_agent: AgentRef | None = Field(None, description="Agent using this control") # TODO: Follow-up with full `used_by_agents` list for richer attribution. used_by_agents_count: int = Field( diff --git a/server/src/agent_control_server/endpoints/agents.py b/server/src/agent_control_server/endpoints/agents.py index 8fc63c32..e2c92cd0 100644 --- a/server/src/agent_control_server/endpoints/agents.py +++ b/server/src/agent_control_server/endpoints/agents.py @@ -4,7 +4,7 @@ from agent_control_engine import list_evaluators from agent_control_models.agent import Agent as APIAgent from agent_control_models.agent import StepSchema -from agent_control_models.controls import ControlDefinitionRuntime +from agent_control_models.controls import ControlDefinition, ControlDefinitionRuntime from agent_control_models.errors import ErrorCode, ValidationErrorItem from agent_control_models.policy import Control as APIControl from agent_control_models.server import ( @@ -112,6 +112,14 @@ def _validate_controls_for_agent(agent: Agent, controls: list[Control]) -> list[ if not control.data: continue + # Skip unrendered template controls — they have no evaluators to validate. + if ( + isinstance(control.data, dict) + and control.data.get("template") is not None + and control.data.get("condition") is None + ): + continue + try: control_definition = ControlDefinitionRuntime.model_validate(control.data) except ValidationError: @@ -167,6 +175,8 @@ def _find_referencing_controls_for_removed_evaluators( referencing_control_set: set[tuple[str, str]] = set() for ctrl in controls: + if not isinstance(ctrl.control, ControlDefinition): + continue # Skip unrendered template controls for _, evaluator_spec in ctrl.control.iter_condition_leaf_parts(): evaluator_ref = evaluator_spec.name if ":" not in evaluator_ref: @@ -228,6 +238,8 @@ async def _build_overwrite_evaluator_removals( references_by_evaluator: dict[str, set[tuple[int, str]]] = {} for control in controls: + if not isinstance(control.control, ControlDefinition): + continue # Skip unrendered template controls for _, evaluator_spec in control.control.iter_condition_leaf_parts(): evaluator_ref = evaluator_spec.name parsed = parse_evaluator_ref_full(evaluator_ref) diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 30cd8efc..839cfd92 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -1,5 +1,5 @@ from agent_control_engine import list_evaluators -from agent_control_models import ControlDefinition, TemplateControlInput +from agent_control_models import ControlDefinition, TemplateControlInput, UnrenderedTemplateControl from agent_control_models.errors import ErrorCode, ValidationErrorItem from agent_control_models.server import ( AgentRef, @@ -39,7 +39,12 @@ from ..models import Agent, AgentData, Control, agent_controls, agent_policies, policy_controls from ..services.condition_traversal import iter_condition_leaves_with_paths from ..services.control_definitions import parse_control_definition_or_api_error -from ..services.control_templates import remap_template_api_error, render_template_control_input +from ..services.control_templates import ( + can_render_template, + remap_template_api_error, + render_template_control_input, + validate_template_structure, +) from ..services.evaluator_utils import ( parse_evaluator_ref_full, validate_config_against_schema, @@ -60,9 +65,11 @@ _logger = get_logger(__name__) -def _serialize_control_definition(control_def: ControlDefinition) -> dict[str, object]: - """Serialize control data for storage while omitting null scope fields.""" - data_json = control_def.model_dump( +def _serialize_control_data( + control_data: ControlDefinition | UnrenderedTemplateControl, +) -> dict[str, object]: + """Serialize control data for JSONB storage.""" + data_json = control_data.model_dump( mode="json", by_alias=True, exclude_none=True, @@ -72,6 +79,10 @@ def _serialize_control_definition(control_def: ControlDefinition) -> dict[str, o data_json["scope"] = { k: v for k, v in data_json["scope"].items() if v is not None } + # Always persist enabled explicitly so _enabled_from_stored_payload reads + # the correct value (especially for unrendered templates where enabled=False). + if "enabled" not in data_json: + data_json["enabled"] = control_data.enabled return data_json @@ -80,6 +91,33 @@ def _is_template_backed_payload(data: object) -> bool: return isinstance(data, dict) and data.get("template") is not None +def _is_unrendered_template(data: object) -> bool: + """Return whether stored control JSON is an unrendered template.""" + return ( + isinstance(data, dict) + and data.get("template") is not None + and data.get("condition") is None + ) + + +def _parse_stored_control_data( + data: dict[str, object], + *, + control_name: str, + control_id: int, +) -> ControlDefinition | UnrenderedTemplateControl: + """Parse stored JSONB into the appropriate model type.""" + if _is_unrendered_template(data): + return UnrenderedTemplateControl.model_validate(data) + + return parse_control_definition_or_api_error( + data, + detail=f"Control '{control_name}' has invalid data", + hint=f"Update the control data using PUT /api/v1/controls/{control_id}/data.", + field_prefix=None, + ) + + def _enabled_from_stored_payload(data: object) -> bool: """Return the persisted enabled flag, defaulting to True when absent.""" if not isinstance(data, dict): @@ -135,16 +173,25 @@ async def _materialize_control_input( db: AsyncSession, current_payload: object | None = None, control_id: int | None = None, -) -> ControlDefinition: - """Resolve raw or template-backed input into a validated control definition.""" +) -> ControlDefinition | UnrenderedTemplateControl: + """Resolve raw or template-backed input into a validated control or unrendered template.""" if isinstance(control_input, TemplateControlInput): - enabled = ( - True if current_payload is None else _enabled_from_stored_payload(current_payload) - ) - return await _render_and_validate_template_input( - control_input, - db=db, - enabled=enabled, + if can_render_template(control_input): + enabled = ( + True if current_payload is None else _enabled_from_stored_payload(current_payload) + ) + return await _render_and_validate_template_input( + control_input, + db=db, + enabled=enabled, + ) + + # Incomplete values — store as unrendered template. + validate_template_structure(control_input.template) + return UnrenderedTemplateControl( + template=control_input.template, + template_values=dict(control_input.template_values), + enabled=False, ) if current_payload is not None and _is_template_backed_payload(current_payload): @@ -381,7 +428,7 @@ async def create_control( ) control_def = await _materialize_control_input(request.data, db=db) - control_data = _serialize_control_definition(control_def) + control_data = _serialize_control_data(control_def) control = Control(name=request.name, data=control_data) db.add(control) @@ -445,11 +492,15 @@ async def get_control( ) # Parse data if present and non-empty - control_data: ControlDefinition | None = None + control_data: ControlDefinition | UnrenderedTemplateControl | None = None if control.data: try: - control_data = ControlDefinition.model_validate(control.data) - except ValidationError: + control_data = _parse_stored_control_data( + control.data, + control_name=control.name, + control_id=control_id, + ) + except Exception: # Data exists but is corrupted - log and return None _logger.warning( "Control '%s' (id=%s) has corrupted data that failed validation", @@ -502,13 +553,12 @@ async def get_control_data( resource_id=str(control_id), hint="Verify the control ID is correct and the control has been created.", ) - control_def = parse_control_definition_or_api_error( + control_data = _parse_stored_control_data( control.data, - detail=f"Control '{control.name}' has invalid data", - hint="Update the control data using PUT /{control_id}/data.", - field_prefix=None, + control_name=control.name, + control_id=control_id, ) - return GetControlDataResponse(data=control_def) + return GetControlDataResponse(data=control_data) @router.put( @@ -559,7 +609,7 @@ async def set_control_data( control_id=control_id, ) - control.data = _serialize_control_definition(control_def) + control.data = _serialize_control_data(control_def) try: await db.commit() except Exception: @@ -595,7 +645,12 @@ async def validate_control_data( Returns: ValidateControlDataResponse with success=True if valid """ - await _materialize_control_input(request.data, db=db) + if isinstance(request.data, TemplateControlInput): + # Validate always attempts full rendering, even if values are incomplete. + # This differs from create/update which allow unrendered storage. + await _render_and_validate_template_input(request.data, db=db, enabled=True) + else: + await _validate_control_definition(request.data, db) return ValidateControlDataResponse(success=True) @@ -809,6 +864,9 @@ async def list_controls( stages=scope.get("stages"), tags=data.get("tags", []), template_backed="template" in data, + template_rendered=( + "condition" in data if "template" in data else None + ), used_by_agent=control_agent_map.get(ctrl.id), used_by_agents_count=len(control_agent_names_map.get(ctrl.id, set())), ) @@ -1059,6 +1117,28 @@ async def patch_control( ], ) + if request.enabled and _is_unrendered_template(control.data): + raise APIValidationError( + error_code=ErrorCode.VALIDATION_ERROR, + detail=( + f"Cannot enable control '{control.name}': " + "unrendered template controls must be rendered first" + ), + resource="Control", + hint=( + f"Provide complete parameter values via PUT /api/v1/controls/{control_id}/data " + "to render the template before enabling." + ), + errors=[ + ValidationErrorItem( + resource="Control", + field="enabled", + code="unrendered_template_cannot_enable", + message="Provide parameter values to render the template before enabling.", + ) + ], + ) + try: ctrl_def = ControlDefinition.model_validate(control.data) if ctrl_def.enabled != request.enabled: diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index 66c965ba..e3dadac7 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -26,6 +26,24 @@ _TEMPLATE_VALUE_MISSING = object() +def can_render_template(template_input: TemplateControlInput) -> bool: + """Return whether the template input has enough values to render. + + True when every required parameter (that has no default) has a value in + ``template_values``. Used to decide between rendered and unrendered + persistence. + """ + template = template_input.template + for name, param in template.parameters.items(): + if not param.required: + continue + has_value = name in template_input.template_values + has_default = getattr(param, "default", None) is not None + if not has_value and not has_default: + return False + return True + + @dataclass(frozen=True) class RenderedTemplateControl: """Rendered template result plus reverse mapping for validation errors.""" @@ -507,6 +525,48 @@ def _reject_agent_scoped_evaluators( ) +def validate_template_structure(template: TemplateDefinition) -> None: + """Validate a template definition's structure without rendering. + + Checks forbidden top-level keys, legacy format, and that ``definition_template`` + is a dict. Used for unrendered template creation where parameter values are + not yet available. + """ + definition_template = template.definition_template + if not isinstance(definition_template, dict): + raise _render_error( + detail="Templates must define a top-level control object", + field="template.definition_template", + code="invalid_definition_template_type", + message="definition_template must be a JSON object representing a control definition.", + ) + + for forbidden_key in ("enabled", "name"): + if forbidden_key in definition_template: + raise _render_error( + detail=f"Templates must not define top-level '{forbidden_key}'", + field=forbidden_key, + code="forbidden_template_field", + message=( + f"Templates must not define top-level '{forbidden_key}'. " + "Manage it outside the template." + ), + ) + + if "condition" not in definition_template and ( + "selector" in definition_template or "evaluator" in definition_template + ): + raise _render_error( + detail="Templates must use the canonical 'condition' format", + field="condition", + code="legacy_condition_format_not_supported", + message=( + "Templates must use the canonical 'condition' wrapper instead of " + "top-level selector/evaluator fields." + ), + ) + + def render_template_control_input( template_input: TemplateControlInput, *, diff --git a/server/src/agent_control_server/services/controls.py b/server/src/agent_control_server/services/controls.py index 44e80020..bb8e371a 100644 --- a/server/src/agent_control_server/services/controls.py +++ b/server/src/agent_control_server/services/controls.py @@ -83,6 +83,18 @@ async def list_controls_for_agent( # Map DB Control to API Control, raising on invalid definitions api_controls: list[APIControl] = [] for c in db_controls: + # Unrendered template controls are returned with their template metadata + if ( + isinstance(c.data, dict) + and c.data.get("template") is not None + and c.data.get("condition") is None + ): + from agent_control_models import UnrenderedTemplateControl + + unrendered = UnrenderedTemplateControl.model_validate(c.data) + api_controls.append(APIControl(id=c.id, name=c.name, control=unrendered)) + continue + context = ( {"allow_invalid_step_name_regex": True} if allow_invalid_step_name_regex @@ -111,6 +123,14 @@ async def list_runtime_controls_for_agent( runtime_controls: list[RuntimeControl] = [] for c in db_controls: + # Skip unrendered template controls — they have no condition to evaluate. + if ( + isinstance(c.data, dict) + and c.data.get("template") is not None + and c.data.get("condition") is None + ): + continue + context = ( {"allow_invalid_step_name_regex": True} if allow_invalid_step_name_regex diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 5ab8eb05..9651a243 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -1456,3 +1456,223 @@ def test_render_control_template_keeps_non_parameterized_errors_on_rendered_fiel and err.get("rendered_field") == "action.decision" for err in body.get("errors", []) ) + + +# ============================================================================= +# Unrendered template control flows +# ============================================================================= + + +def _unrendered_template_payload() -> dict[str, object]: + """Template payload with no template_values — creates an unrendered control.""" + return { + "template": { + "description": "Regex denial template", + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + }, + }, + "definition_template": { + "description": "Template-backed control", + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {}, + } + + +def _create_unrendered_control(client: TestClient) -> tuple[int, str]: + control_name = f"unrendered-control-{uuid.uuid4()}" + response = client.put( + "/api/v1/controls", + json={"name": control_name, "data": _unrendered_template_payload()}, + ) + assert response.status_code == 200, response.text + return response.json()["control_id"], control_name + + +def test_create_unrendered_template_control_without_values(client: TestClient) -> None: + # Given: a template payload with no template_values + + # When: creating a control + control_id, _ = _create_unrendered_control(client) + + # Then: the control is created and stored as unrendered + get_response = client.get(f"/api/v1/controls/{control_id}/data") + assert get_response.status_code == 200, get_response.text + data = get_response.json()["data"] + assert data["template"]["description"] == "Regex denial template" + assert data["template_values"] == {} + assert data["enabled"] is False + + # And: the stored data has no rendered fields + assert "condition" not in data + assert "action" not in data + assert "execution" not in data + + +def test_get_control_returns_unrendered_template_metadata(client: TestClient) -> None: + # Given: an unrendered template control + control_id, control_name = _create_unrendered_control(client) + + # When: getting the control by ID + response = client.get(f"/api/v1/controls/{control_id}") + + # Then: the response includes template metadata but no rendered fields + assert response.status_code == 200, response.text + body = response.json() + assert body["name"] == control_name + assert body["data"]["template"]["parameters"]["pattern"]["type"] == "regex_re2" + assert body["data"]["enabled"] is False + assert "condition" not in body["data"] + + +def test_update_unrendered_template_with_complete_values_renders(client: TestClient) -> None: + # Given: an unrendered template control + control_id, _ = _create_unrendered_control(client) + + # When: updating with complete template values + rendered_payload = _template_payload() + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": rendered_payload}, + ) + + # Then: the control is now rendered + assert put_response.status_code == 200, put_response.text + get_response = client.get(f"/api/v1/controls/{control_id}/data") + assert get_response.status_code == 200, get_response.text + data = get_response.json()["data"] + assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" + assert data["template_values"]["pattern"] == "hello" + # And: enabled remains false (preserved from unrendered state) + assert data["enabled"] is False + + +def test_update_unrendered_template_with_still_incomplete_values_stays_unrendered( + client: TestClient, +) -> None: + # Given: an unrendered template control + control_id, _ = _create_unrendered_control(client) + + # When: updating with still-empty values + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": _unrendered_template_payload()}, + ) + + # Then: the control stays unrendered + assert put_response.status_code == 200, put_response.text + get_response = client.get(f"/api/v1/controls/{control_id}/data") + data = get_response.json()["data"] + assert data["enabled"] is False + assert "condition" not in data + + +def test_patch_enable_on_unrendered_template_is_rejected(client: TestClient) -> None: + # Given: an unrendered template control + control_id, _ = _create_unrendered_control(client) + + # When: trying to enable it + response = client.patch( + f"/api/v1/controls/{control_id}", + json={"enabled": True}, + ) + + # Then: the server rejects with 422 + assert response.status_code == 422 + body = response.json() + assert any( + err.get("code") == "unrendered_template_cannot_enable" + for err in body.get("errors", []) + ) + + +def test_patch_name_on_unrendered_template_works(client: TestClient) -> None: + # Given: an unrendered template control + control_id, _ = _create_unrendered_control(client) + new_name = f"renamed-unrendered-{uuid.uuid4()}" + + # When: renaming it + response = client.patch( + f"/api/v1/controls/{control_id}", + json={"name": new_name}, + ) + + # Then: the rename succeeds + assert response.status_code == 200, response.text + assert response.json()["name"] == new_name + + +def test_unrendered_template_excluded_from_evaluation(client: TestClient) -> None: + # Given: an unrendered template control attached to an agent + control_id, _ = _create_unrendered_control(client) + agent_name = _assign_control_to_agent(client, control_id) + + # When: evaluating a step + eval_response = _evaluate_step( + client, agent_name, step_name="any-step", input_value="hello", + ) + + # Then: evaluation succeeds (unrendered control is skipped) + assert eval_response.status_code == 200, eval_response.text + assert eval_response.json()["is_safe"] is True + + +def test_unrendered_template_shows_in_list_with_correct_flags(client: TestClient) -> None: + # Given: an unrendered template control + control_id, _ = _create_unrendered_control(client) + + # When: listing controls + response = client.get("/api/v1/controls", params={"template_backed": True}) + + # Then: the control appears with template_backed=True and template_rendered=False + assert response.status_code == 200, response.text + controls = response.json()["controls"] + unrendered = next((c for c in controls if c["id"] == control_id), None) + assert unrendered is not None + assert unrendered["template_backed"] is True + assert unrendered["template_rendered"] is False + + +def test_unrendered_template_can_be_deleted(client: TestClient) -> None: + # Given: an unrendered template control + control_id, _ = _create_unrendered_control(client) + + # When: deleting it + response = client.delete(f"/api/v1/controls/{control_id}", params={"force": True}) + + # Then: deletion succeeds + assert response.status_code == 200, response.text + assert response.json()["success"] is True + + +def test_rendered_template_then_update_to_unrendered_stays_rendered( + client: TestClient, +) -> None: + # Given: a rendered template control + control_id = _create_template_control(client) + + # When: updating with empty values (attempting to "un-render") + put_response = client.put( + f"/api/v1/controls/{control_id}/data", + json={"data": _unrendered_template_payload()}, + ) + + # Then: the control becomes unrendered (stored without rendered fields) + assert put_response.status_code == 200, put_response.text + get_response = client.get(f"/api/v1/controls/{control_id}/data") + data = get_response.json()["data"] + assert data["enabled"] is False + assert "condition" not in data From b2e940d82cd9851982759be9b79f803e5127a739 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 14:28:10 -0700 Subject: [PATCH 18/25] fix: harden unrendered template controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Prevent rendered→unrendered downgrade: updating a rendered template control with incomplete values now forces a full render attempt, returning a clear error about missing parameters instead of silently stripping rendered fields - Deepen unrendered structural validation: validate_template_structure now walks definition_template to check $param bindings, reject undefined parameter references, detect unused parameters, and reject hardcoded agent-scoped evaluator names - Fix PATCH enabled=false on unrendered templates: detect unrendered state before attempting ControlDefinition.model_validate, treating disable as a no-op instead of raising CORRUPTED_DATA - Add 4 behavioral tests: rendered rejects incomplete update, PATCH disable no-op, unrendered rejects undefined $param / unused param / agent-scoped evaluator --- .../endpoints/controls.py | 128 +++++++++++------- .../services/control_templates.py | 119 +++++++++++++++- server/tests/test_control_templates.py | 104 +++++++++++++- 3 files changed, 291 insertions(+), 60 deletions(-) diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 839cfd92..6732c4fa 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -186,7 +186,24 @@ async def _materialize_control_input( enabled=enabled, ) - # Incomplete values — store as unrendered template. + # Incomplete values — only allowed for new controls or already-unrendered + # templates. Updating a rendered control with incomplete values is + # rejected to prevent silently stripping rendered fields. + current_is_rendered = ( + current_payload is not None + and isinstance(current_payload, dict) + and current_payload.get("condition") is not None + ) + if current_is_rendered: + # Force a full render attempt so the caller gets a clear error + # about which required parameters are missing. + enabled = _enabled_from_stored_payload(current_payload) + return await _render_and_validate_template_input( + control_input, + db=db, + enabled=enabled, + ) + validate_template_structure(control_input.template) return UnrenderedTemplateControl( template=control_input.template, @@ -1117,57 +1134,64 @@ async def patch_control( ], ) - if request.enabled and _is_unrendered_template(control.data): - raise APIValidationError( - error_code=ErrorCode.VALIDATION_ERROR, - detail=( - f"Cannot enable control '{control.name}': " - "unrendered template controls must be rendered first" - ), - resource="Control", - hint=( - f"Provide complete parameter values via PUT /api/v1/controls/{control_id}/data " - "to render the template before enabling." - ), - errors=[ - ValidationErrorItem( - resource="Control", - field="enabled", - code="unrendered_template_cannot_enable", - message="Provide parameter values to render the template before enabling.", - ) - ], - ) - - try: - ctrl_def = ControlDefinition.model_validate(control.data) - if ctrl_def.enabled != request.enabled: - new_data = dict(control.data) - new_data["enabled"] = request.enabled - control.data = new_data - updated = True - current_enabled = request.enabled if updated else ctrl_def.enabled - except ValidationError: - _logger.error( - "Control '%s' (%s) has corrupted data in patch request", - control.name, - control_id, - exc_info=True, - ) - raise APIValidationError( - error_code=ErrorCode.CORRUPTED_DATA, - detail=f"Control '{control.name}' has corrupted data", - resource="Control", - hint="Update the control data using PUT /{control_id}/data.", - errors=[ - ValidationErrorItem( - resource="Control", - field="data", - code="corrupted_data", - message=_CORRUPTED_CONTROL_DATA_MESSAGE, - ) - ], - ) + if _is_unrendered_template(control.data): + if request.enabled: + raise APIValidationError( + error_code=ErrorCode.VALIDATION_ERROR, + detail=( + f"Cannot enable control '{control.name}': " + "unrendered template controls must be rendered first" + ), + resource="Control", + hint=( + "Provide complete parameter values via " + f"PUT /api/v1/controls/{control_id}/data " + "to render the template before enabling." + ), + errors=[ + ValidationErrorItem( + resource="Control", + field="enabled", + code="unrendered_template_cannot_enable", + message=( + "Provide parameter values to render the " + "template before enabling." + ), + ) + ], + ) + # enabled=False on an unrendered template is a no-op (already false). + current_enabled = False + else: + try: + ctrl_def = ControlDefinition.model_validate(control.data) + if ctrl_def.enabled != request.enabled: + new_data = dict(control.data) + new_data["enabled"] = request.enabled + control.data = new_data + updated = True + current_enabled = request.enabled if updated else ctrl_def.enabled + except ValidationError: + _logger.error( + "Control '%s' (%s) has corrupted data in patch request", + control.name, + control_id, + exc_info=True, + ) + raise APIValidationError( + error_code=ErrorCode.CORRUPTED_DATA, + detail=f"Control '{control.name}' has corrupted data", + resource="Control", + hint="Update the control data using PUT /{control_id}/data.", + errors=[ + ValidationErrorItem( + resource="Control", + field="data", + code="corrupted_data", + message=_CORRUPTED_CONTROL_DATA_MESSAGE, + ) + ], + ) elif control.data: # Get current enabled status for response try: diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index e3dadac7..a34b691e 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -525,12 +525,57 @@ def _reject_agent_scoped_evaluators( ) +def _collect_param_references( + value: JsonValue, + *, + path_parts: list[str | int], + template: TemplateDefinition, + referenced: set[str], +) -> None: + """Walk definition_template collecting $param references and validating bindings.""" + if isinstance(value, dict): + if "$param" in value: + if len(value) != 1 or not isinstance(value["$param"], str): + raise _render_error( + detail="Invalid $param binding object in template definition", + field=_format_rendered_path(path_parts), + code="invalid_param_binding", + message="A $param binding must be exactly {'$param': ''}.", + ) + parameter_name = value["$param"] + if parameter_name not in template.parameters: + raise _render_error( + detail=f"Template references undefined parameter '{parameter_name}'", + field=_format_rendered_path(path_parts), + code="undefined_parameter_reference", + message=f"Template references undefined parameter '{parameter_name}'.", + ) + referenced.add(parameter_name) + return + + for key, nested in value.items(): + _collect_param_references( + nested, + path_parts=[*path_parts, key], + template=template, + referenced=referenced, + ) + elif isinstance(value, list): + for idx, nested in enumerate(value): + _collect_param_references( + nested, + path_parts=[*path_parts, idx], + template=template, + referenced=referenced, + ) + + def validate_template_structure(template: TemplateDefinition) -> None: """Validate a template definition's structure without rendering. - Checks forbidden top-level keys, legacy format, and that ``definition_template`` - is a dict. Used for unrendered template creation where parameter values are - not yet available. + Performs all structural checks that don't require parameter values: + forbidden top-level keys, legacy format, $param reference validity, + unused parameter detection, and agent-scoped evaluator rejection. """ definition_template = template.definition_template if not isinstance(definition_template, dict): @@ -566,6 +611,74 @@ def validate_template_structure(template: TemplateDefinition) -> None: ), ) + # Walk the template to validate $param references and collect referenced params. + referenced: set[str] = set() + _collect_param_references( + definition_template, + path_parts=[], + template=template, + referenced=referenced, + ) + + # Reject unused parameters. + unused = sorted(set(template.parameters) - referenced) + if unused: + raise APIValidationError( + error_code=ErrorCode.TEMPLATE_RENDER_ERROR, + detail="Template defines parameters that are never referenced", + resource="Control", + hint="Remove unused parameters or reference them in definition_template.", + errors=[ + ValidationErrorItem( + resource="Control", + field=f"template.parameters.{name}", + code="unused_template_parameter", + message=f"Template parameter '{name}' is never referenced.", + parameter=name, + parameter_label=template.parameters[name].label, + ) + for name in unused + ], + ) + + # Reject agent-scoped evaluator names baked into the template (not via $param). + _reject_hardcoded_agent_scoped_evaluators(definition_template) + + +def _reject_hardcoded_agent_scoped_evaluators( + definition_template: dict[str, JsonValue], +) -> None: + """Reject agent-scoped evaluator names that are hardcoded in the template.""" + # Walk condition tree looking for evaluator.name fields containing ':' + condition = definition_template.get("condition") + if not isinstance(condition, dict): + return + + stack: list[dict[str, JsonValue]] = [condition] + while stack: + node = stack.pop() + evaluator = node.get("evaluator") + if isinstance(evaluator, dict): + name = evaluator.get("name") + if isinstance(name, str) and ":" in name: + raise _render_error( + detail="Agent-scoped evaluators are not supported in control templates", + field="condition.evaluator.name", + code="agent_scoped_evaluator_not_supported", + message="Agent-scoped evaluators are not supported in control templates.", + ) + + for key in ("and", "or"): + children = node.get(key) + if isinstance(children, list): + for child in children: + if isinstance(child, dict): + stack.append(child) + + not_child = node.get("not") + if isinstance(not_child, dict): + stack.append(not_child) + def render_template_control_input( template_input: TemplateControlInput, diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 9651a243..54eb83b4 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -1658,7 +1658,97 @@ def test_unrendered_template_can_be_deleted(client: TestClient) -> None: assert response.json()["success"] is True -def test_rendered_template_then_update_to_unrendered_stays_rendered( +def test_patch_disable_on_unrendered_template_is_noop(client: TestClient) -> None: + # Given: an unrendered template control (already enabled=false) + control_id, _ = _create_unrendered_control(client) + + # When: patching enabled=false (redundant, but should not crash) + response = client.patch( + f"/api/v1/controls/{control_id}", + json={"enabled": False}, + ) + + # Then: the request succeeds without error (no CORRUPTED_DATA) + assert response.status_code == 200, response.text + assert response.json()["enabled"] is False + + +def test_create_unrendered_template_rejects_undefined_param_reference( + client: TestClient, +) -> None: + # Given: a template whose definition_template references an undefined parameter + payload = _unrendered_template_payload() + payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["extra"] = { # type: ignore[index] + "$param": "nonexistent", + } + + # When: creating the unrendered control + response = client.put( + "/api/v1/controls", + json={"name": f"bad-unrendered-{uuid.uuid4()}", "data": payload}, + ) + + # Then: the server rejects with a structural validation error + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("code") == "undefined_parameter_reference" + for err in body.get("errors", []) + ) + + +def test_create_unrendered_template_rejects_unused_parameter( + client: TestClient, +) -> None: + # Given: a template with an extra parameter never referenced in definition_template + payload = _unrendered_template_payload() + payload["template"]["parameters"]["unused_param"] = { # type: ignore[index] + "type": "string", + "label": "Unused", + "default": "val", + } + + # When: creating the unrendered control + response = client.put( + "/api/v1/controls", + json={"name": f"unused-param-{uuid.uuid4()}", "data": payload}, + ) + + # Then: the server rejects with unused parameter error + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("code") == "unused_template_parameter" + for err in body.get("errors", []) + ) + + +def test_create_unrendered_template_rejects_agent_scoped_evaluator( + client: TestClient, +) -> None: + # Given: a template with a hardcoded agent-scoped evaluator name + payload = _unrendered_template_payload() + payload["template"]["definition_template"]["condition"]["evaluator"]["name"] = "agent-x:custom" # type: ignore[index] + + # When: creating the unrendered control + response = client.put( + "/api/v1/controls", + json={"name": f"agent-scoped-{uuid.uuid4()}", "data": payload}, + ) + + # Then: the server rejects + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("code") == "agent_scoped_evaluator_not_supported" + for err in body.get("errors", []) + ) + + +def test_rendered_template_rejects_update_with_incomplete_values( client: TestClient, ) -> None: # Given: a rendered template control @@ -1670,9 +1760,13 @@ def test_rendered_template_then_update_to_unrendered_stays_rendered( json={"data": _unrendered_template_payload()}, ) - # Then: the control becomes unrendered (stored without rendered fields) - assert put_response.status_code == 200, put_response.text + # Then: the server rejects (missing required parameter for a rendered control) + assert put_response.status_code == 422 + body = put_response.json() + assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" + + # And: the control remains rendered and unchanged get_response = client.get(f"/api/v1/controls/{control_id}/data") data = get_response.json()["data"] - assert data["enabled"] is False - assert "condition" not in data + assert "condition" in data + assert data["condition"]["evaluator"]["config"]["pattern"] == "hello" From 817c3dbcb43dbb5006d79adb35ed3e7dc3a0633c Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 14:33:54 -0700 Subject: [PATCH 19/25] fix: close remaining unrendered template validation gaps - Reject optional params without defaults in unrendered structural validation (catches templates that can never render at creation time) - Fix PATCH rename-only on unrendered templates: detect unrendered state before ControlDefinition.model_validate to avoid false CORRUPTED_DATA - Export UnrenderedTemplateControl from Python SDK - Strengthen rename test to verify enabled=false in response - Add test for optional-param-without-default rejection on unrendered create --- sdks/python/src/agent_control/__init__.py | 2 ++ .../endpoints/controls.py | 17 ++++++---- .../services/control_templates.py | 19 +++++++++++ server/tests/test_control_templates.py | 34 +++++++++++++++++-- 4 files changed, 63 insertions(+), 9 deletions(-) diff --git a/sdks/python/src/agent_control/__init__.py b/sdks/python/src/agent_control/__init__.py index 65e7c64e..649ab69d 100644 --- a/sdks/python/src/agent_control/__init__.py +++ b/sdks/python/src/agent_control/__init__.py @@ -70,6 +70,7 @@ async def handle_input(user_message: str) -> str: TemplateControlInput, TemplateDefinition, TemplateValue, + UnrenderedTemplateControl, ) from . import agents, controls, evaluation, evaluators, policies @@ -1371,6 +1372,7 @@ async def main(): "ControlDefinition", "TemplateControlInput", "TemplateDefinition", + "UnrenderedTemplateControl", "ControlSelector", "ControlScope", "ControlAction", diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 6732c4fa..12237082 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -1194,12 +1194,17 @@ async def patch_control( ) elif control.data: # Get current enabled status for response - try: - ctrl_def = ControlDefinition.model_validate(control.data) - current_enabled = ctrl_def.enabled - except ValidationError: - # Data corrupted, use default enabled=True - _logger.warning("Control '%s' has invalid data, using default", control.name) + if _is_unrendered_template(control.data): + current_enabled = False + else: + try: + ctrl_def = ControlDefinition.model_validate(control.data) + current_enabled = ctrl_def.enabled + except ValidationError: + _logger.warning( + "Control '%s' has invalid data, using default", + control.name, + ) # Commit if anything changed if updated: diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index a34b691e..1c65c9ad 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -550,6 +550,25 @@ def _collect_param_references( code="undefined_parameter_reference", message=f"Template references undefined parameter '{parameter_name}'.", ) + # Reject optional params without defaults — they can never render. + param_def = template.parameters[parameter_name] + if ( + not param_def.required + and _parameter_default(param_def) is _TEMPLATE_VALUE_MISSING + ): + raise _render_error( + detail=( + f"Template parameter '{parameter_name}' is optional " + "but referenced without a default value" + ), + field=f"template.parameters.{parameter_name}", + code="optional_referenced_parameter_requires_default", + message=( + f"Optional template parameter '{param_def.label}' is " + "referenced in the template and must define a default " + "value or be marked required." + ), + ) referenced.add(parameter_name) return diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 54eb83b4..b0b5210f 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -1599,7 +1599,9 @@ def test_patch_enable_on_unrendered_template_is_rejected(client: TestClient) -> ) -def test_patch_name_on_unrendered_template_works(client: TestClient) -> None: +def test_patch_name_on_unrendered_template_returns_correct_enabled( + client: TestClient, +) -> None: # Given: an unrendered template control control_id, _ = _create_unrendered_control(client) new_name = f"renamed-unrendered-{uuid.uuid4()}" @@ -1610,9 +1612,35 @@ def test_patch_name_on_unrendered_template_works(client: TestClient) -> None: json={"name": new_name}, ) - # Then: the rename succeeds + # Then: the rename succeeds and enabled is correctly reported as False assert response.status_code == 200, response.text - assert response.json()["name"] == new_name + body = response.json() + assert body["name"] == new_name + assert body["enabled"] is False + + +def test_create_unrendered_template_rejects_optional_param_without_default( + client: TestClient, +) -> None: + # Given: a template with an optional parameter that has no default but is + # referenced in definition_template + payload = _unrendered_template_payload() + payload["template"]["parameters"]["pattern"]["required"] = False # type: ignore[index] + + # When: creating an unrendered control + response = client.put( + "/api/v1/controls", + json={"name": f"bad-optional-{uuid.uuid4()}", "data": payload}, + ) + + # Then: the server rejects — this template can never render + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" + assert any( + err.get("code") == "optional_referenced_parameter_requires_default" + for err in body.get("errors", []) + ) def test_unrendered_template_excluded_from_evaluation(client: TestClient) -> None: From 124aaac5b82e15a6961dc481d3f0e71b2b5ed3bb Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 14:43:02 -0700 Subject: [PATCH 20/25] chore: regenerate TypeScript SDK for unrendered template types --- .../src/generated/models/control-summary.ts | 6 ++ .../src/generated/models/control.ts | 48 +++++++++---- .../models/get-control-data-response.ts | 45 +++++++++--- .../generated/models/get-control-response.ts | 44 +++++++++++- sdks/typescript/src/generated/models/index.ts | 1 + .../models/unrendered-template-control.ts | 71 +++++++++++++++++++ 6 files changed, 192 insertions(+), 23 deletions(-) create mode 100644 sdks/typescript/src/generated/models/unrendered-template-control.ts diff --git a/sdks/typescript/src/generated/models/control-summary.ts b/sdks/typescript/src/generated/models/control-summary.ts index c9c717a4..4c0b0fb3 100644 --- a/sdks/typescript/src/generated/models/control-summary.ts +++ b/sdks/typescript/src/generated/models/control-summary.ts @@ -50,6 +50,10 @@ export type ControlSummary = { * Whether the control was created from a template */ templateBacked: boolean; + /** + * Whether a template-backed control has been rendered. True for rendered templates, False for unrendered templates, None for non-template controls. + */ + templateRendered?: boolean | null | undefined; /** * Agent using this control */ @@ -75,6 +79,7 @@ export const ControlSummary$inboundSchema: z.ZodMiniType< step_types: z.optional(z.nullable(z.array(types.string()))), tags: types.optional(z.array(types.string())), template_backed: z._default(types.boolean(), false), + template_rendered: z.optional(z.nullable(types.boolean())), used_by_agent: z.optional(z.nullable(AgentRef$inboundSchema)), used_by_agents_count: z._default(types.number(), 0), }), @@ -82,6 +87,7 @@ export const ControlSummary$inboundSchema: z.ZodMiniType< return remap$(v, { "step_types": "stepTypes", "template_backed": "templateBacked", + "template_rendered": "templateRendered", "used_by_agent": "usedByAgent", "used_by_agents_count": "usedByAgentsCount", }); diff --git a/sdks/typescript/src/generated/models/control.ts b/sdks/typescript/src/generated/models/control.ts index f1fc4616..396c6970 100644 --- a/sdks/typescript/src/generated/models/control.ts +++ b/sdks/typescript/src/generated/models/control.ts @@ -6,37 +6,61 @@ import * as z from "zod/v4-mini"; import { safeParse } from "../lib/schemas.js"; import { Result as SafeParseResult } from "../types/fp.js"; import * as types from "../types/primitives.js"; +import { smartUnion } from "../types/smart-union.js"; import { ControlDefinitionOutput, ControlDefinitionOutput$inboundSchema, } from "./control-definition-output.js"; import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + UnrenderedTemplateControl, + UnrenderedTemplateControl$inboundSchema, +} from "./unrendered-template-control.js"; + +export type ControlControl = + | ControlDefinitionOutput + | UnrenderedTemplateControl; /** * A control with identity and configuration. * * @remarks * - * Note: Only fully-configured controls (with valid ControlDefinition) - * are returned from API endpoints. Unconfigured controls are filtered out. + * For rendered controls (raw or template-backed), ``control`` is a + * ``ControlDefinition``. For unrendered template controls, ``control`` + * is an ``UnrenderedTemplateControl``. */ export type Control = { - /** - * A control definition to evaluate agent interactions. - * - * @remarks - * - * This model contains only the logic and configuration. - * Identity fields (id, name) are managed by the database. - */ - control: ControlDefinitionOutput; + control: ControlDefinitionOutput | UnrenderedTemplateControl; id: number; name: string; }; +/** @internal */ +export const ControlControl$inboundSchema: z.ZodMiniType< + ControlControl, + unknown +> = smartUnion([ + ControlDefinitionOutput$inboundSchema, + UnrenderedTemplateControl$inboundSchema, +]); + +export function controlControlFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => ControlControl$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'ControlControl' from JSON`, + ); +} + /** @internal */ export const Control$inboundSchema: z.ZodMiniType = z.object({ - control: ControlDefinitionOutput$inboundSchema, + control: smartUnion([ + ControlDefinitionOutput$inboundSchema, + UnrenderedTemplateControl$inboundSchema, + ]), id: types.number(), name: types.string(), }); diff --git a/sdks/typescript/src/generated/models/get-control-data-response.ts b/sdks/typescript/src/generated/models/get-control-data-response.ts index 5f452381..11ed429f 100644 --- a/sdks/typescript/src/generated/models/get-control-data-response.ts +++ b/sdks/typescript/src/generated/models/get-control-data-response.ts @@ -5,30 +5,59 @@ import * as z from "zod/v4-mini"; import { safeParse } from "../lib/schemas.js"; import { Result as SafeParseResult } from "../types/fp.js"; +import { smartUnion } from "../types/smart-union.js"; import { ControlDefinitionOutput, ControlDefinitionOutput$inboundSchema, } from "./control-definition-output.js"; import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + UnrenderedTemplateControl, + UnrenderedTemplateControl$inboundSchema, +} from "./unrendered-template-control.js"; + +/** + * Control data payload (rendered control or unrendered template) + */ +export type GetControlDataResponseData = + | ControlDefinitionOutput + | UnrenderedTemplateControl; export type GetControlDataResponse = { /** - * A control definition to evaluate agent interactions. - * - * @remarks - * - * This model contains only the logic and configuration. - * Identity fields (id, name) are managed by the database. + * Control data payload (rendered control or unrendered template) */ - data: ControlDefinitionOutput; + data: ControlDefinitionOutput | UnrenderedTemplateControl; }; +/** @internal */ +export const GetControlDataResponseData$inboundSchema: z.ZodMiniType< + GetControlDataResponseData, + unknown +> = smartUnion([ + ControlDefinitionOutput$inboundSchema, + UnrenderedTemplateControl$inboundSchema, +]); + +export function getControlDataResponseDataFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => GetControlDataResponseData$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'GetControlDataResponseData' from JSON`, + ); +} + /** @internal */ export const GetControlDataResponse$inboundSchema: z.ZodMiniType< GetControlDataResponse, unknown > = z.object({ - data: ControlDefinitionOutput$inboundSchema, + data: smartUnion([ + ControlDefinitionOutput$inboundSchema, + UnrenderedTemplateControl$inboundSchema, + ]), }); export function getControlDataResponseFromJSON( diff --git a/sdks/typescript/src/generated/models/get-control-response.ts b/sdks/typescript/src/generated/models/get-control-response.ts index 9ce485d6..bf67a1c9 100644 --- a/sdks/typescript/src/generated/models/get-control-response.ts +++ b/sdks/typescript/src/generated/models/get-control-response.ts @@ -6,20 +6,32 @@ import * as z from "zod/v4-mini"; import { safeParse } from "../lib/schemas.js"; import { Result as SafeParseResult } from "../types/fp.js"; import * as types from "../types/primitives.js"; +import { smartUnion } from "../types/smart-union.js"; import { ControlDefinitionOutput, ControlDefinitionOutput$inboundSchema, } from "./control-definition-output.js"; import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + UnrenderedTemplateControl, + UnrenderedTemplateControl$inboundSchema, +} from "./unrendered-template-control.js"; + +/** + * Control configuration data. A ControlDefinition for raw/rendered controls, an UnrenderedTemplateControl for unrendered templates, or None if not yet configured. + */ +export type GetControlResponseData = + | ControlDefinitionOutput + | UnrenderedTemplateControl; /** * Response containing control details. */ export type GetControlResponse = { /** - * Control configuration data (None if not yet configured) + * Control configuration data. A ControlDefinition for raw/rendered controls, an UnrenderedTemplateControl for unrendered templates, or None if not yet configured. */ - data?: ControlDefinitionOutput | null | undefined; + data?: ControlDefinitionOutput | UnrenderedTemplateControl | null | undefined; /** * Control ID */ @@ -30,12 +42,38 @@ export type GetControlResponse = { name: string; }; +/** @internal */ +export const GetControlResponseData$inboundSchema: z.ZodMiniType< + GetControlResponseData, + unknown +> = smartUnion([ + ControlDefinitionOutput$inboundSchema, + UnrenderedTemplateControl$inboundSchema, +]); + +export function getControlResponseDataFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => GetControlResponseData$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'GetControlResponseData' from JSON`, + ); +} + /** @internal */ export const GetControlResponse$inboundSchema: z.ZodMiniType< GetControlResponse, unknown > = z.object({ - data: z.optional(z.nullable(ControlDefinitionOutput$inboundSchema)), + data: z.optional( + z.nullable( + smartUnion([ + ControlDefinitionOutput$inboundSchema, + UnrenderedTemplateControl$inboundSchema, + ]), + ), + ), id: types.number(), name: types.string(), }); diff --git a/sdks/typescript/src/generated/models/index.ts b/sdks/typescript/src/generated/models/index.ts index 72eeab0a..35fa9695 100644 --- a/sdks/typescript/src/generated/models/index.ts +++ b/sdks/typescript/src/generated/models/index.ts @@ -89,6 +89,7 @@ export * from "./template-definition-output.js"; export * from "./template-parameter-definition.js"; export * from "./template-value.js"; export * from "./timeseries-bucket.js"; +export * from "./unrendered-template-control.js"; export * from "./validate-control-data-request.js"; export * from "./validate-control-data-response.js"; export * from "./validation-error.js"; diff --git a/sdks/typescript/src/generated/models/unrendered-template-control.ts b/sdks/typescript/src/generated/models/unrendered-template-control.ts new file mode 100644 index 00000000..92d7f221 --- /dev/null +++ b/sdks/typescript/src/generated/models/unrendered-template-control.ts @@ -0,0 +1,71 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; +import { safeParse } from "../lib/schemas.js"; +import { Result as SafeParseResult } from "../types/fp.js"; +import * as types from "../types/primitives.js"; +import { SDKValidationError } from "./errors/sdk-validation-error.js"; +import { + TemplateDefinitionOutput, + TemplateDefinitionOutput$inboundSchema, +} from "./template-definition-output.js"; +import { + TemplateValue, + TemplateValue$inboundSchema, +} from "./template-value.js"; + +/** + * Stored state of a template control that hasn't been rendered yet. + * + * @remarks + * + * An unrendered template has a template definition and possibly partial + * parameter values, but no concrete condition/action/execution fields. + * It is always ``enabled=False`` and excluded from evaluation. + */ +export type UnrenderedTemplateControl = { + /** + * Unrendered templates are always disabled + */ + enabled: false; + /** + * Reusable template with typed parameters and a JSON definition template. + */ + template: TemplateDefinitionOutput; + /** + * Partial or empty parameter values + */ + templateValues?: { [k: string]: TemplateValue } | undefined; +}; + +/** @internal */ +export const UnrenderedTemplateControl$inboundSchema: z.ZodMiniType< + UnrenderedTemplateControl, + unknown +> = z.pipe( + z.object({ + enabled: z._default(types.literal(false), false), + template: TemplateDefinitionOutput$inboundSchema, + template_values: types.optional( + z.record(z.string(), TemplateValue$inboundSchema), + ), + }), + z.transform((v) => { + return remap$(v, { + "template_values": "templateValues", + }); + }), +); + +export function unrenderedTemplateControlFromJSON( + jsonString: string, +): SafeParseResult { + return safeParse( + jsonString, + (x) => UnrenderedTemplateControl$inboundSchema.parse(JSON.parse(x)), + `Failed to parse 'UnrenderedTemplateControl' from JSON`, + ); +} From 5b0a5301d52bf08388e8b4af84b4df0571063f59 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 14:54:47 -0700 Subject: [PATCH 21/25] fix: validate partial template values and polish unrendered paths - Reject unknown keys and wrong-typed values in partial template_values on unrendered create (fail fast instead of persisting garbage) - Deduplicate structural validation: render_template_control_input now calls validate_template_structure instead of inlining the same checks - Fix description fallback in list endpoint: unrendered templates show template.description when top-level description is absent - Fix _reject_hardcoded_agent_scoped_evaluators to report actual condition path instead of hardcoded "condition.evaluator.name" - Fix PATCH error message indentation - Move UnrenderedTemplateControl import to module level in controls service - Add tests: unknown value key rejection, wrong-type value rejection, description fallback in list --- .../endpoints/controls.py | 15 +++- .../services/control_templates.py | 82 ++++++++++--------- .../agent_control_server/services/controls.py | 4 +- server/tests/test_control_templates.py | 64 +++++++++++++++ 4 files changed, 121 insertions(+), 44 deletions(-) diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 12237082..3fb1c0f2 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -43,6 +43,7 @@ can_render_template, remap_template_api_error, render_template_control_input, + validate_partial_template_values, validate_template_structure, ) from ..services.evaluator_utils import ( @@ -205,6 +206,9 @@ async def _materialize_control_input( ) validate_template_structure(control_input.template) + validate_partial_template_values( + control_input.template, control_input.template_values, + ) return UnrenderedTemplateControl( template=control_input.template, template_values=dict(control_input.template_values), @@ -874,7 +878,10 @@ async def list_controls( ControlSummary( id=ctrl.id, name=ctrl.name, - description=data.get("description"), + description=( + data.get("description") + or (data.get("template") or {}).get("description") + ), enabled=data.get("enabled", True), execution=data.get("execution"), step_types=scope.get("step_types"), @@ -1154,9 +1161,9 @@ async def patch_control( field="enabled", code="unrendered_template_cannot_enable", message=( - "Provide parameter values to render the " - "template before enabling." - ), + "Provide parameter values to render " + "the template before enabling." + ), ) ], ) diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index 1c65c9ad..0ab2d9b3 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -664,25 +664,59 @@ def validate_template_structure(template: TemplateDefinition) -> None: _reject_hardcoded_agent_scoped_evaluators(definition_template) +def validate_partial_template_values( + template: TemplateDefinition, + template_values: Mapping[str, TemplateValue], +) -> None: + """Validate provided template values without requiring completeness. + + Rejects unknown parameter keys and type-checks any values that are + provided. Called for unrendered template creation so invalid values + fail fast instead of persisting silently. + """ + unknown_keys = sorted(set(template_values) - set(template.parameters)) + if unknown_keys: + raise APIValidationError( + error_code=ErrorCode.TEMPLATE_PARAMETER_INVALID, + detail="Unknown template parameter(s) supplied", + resource="Control", + hint="Remove unknown template parameter keys and try again.", + errors=[ + ValidationErrorItem( + resource="Control", + field=f"template_values.{name}", + code="unknown_parameter", + message=f"Unknown template parameter '{name}'.", + parameter=name, + ) + for name in unknown_keys + ], + ) + + for name, value in template_values.items(): + if name in template.parameters: + _coerce_parameter_value(name, template.parameters[name], value) + + def _reject_hardcoded_agent_scoped_evaluators( definition_template: dict[str, JsonValue], ) -> None: """Reject agent-scoped evaluator names that are hardcoded in the template.""" - # Walk condition tree looking for evaluator.name fields containing ':' condition = definition_template.get("condition") if not isinstance(condition, dict): return - stack: list[dict[str, JsonValue]] = [condition] + # Walk condition tree tracking the path for accurate error reporting. + stack: list[tuple[dict[str, JsonValue], str]] = [(condition, "condition")] while stack: - node = stack.pop() + node, path = stack.pop() evaluator = node.get("evaluator") if isinstance(evaluator, dict): name = evaluator.get("name") if isinstance(name, str) and ":" in name: raise _render_error( detail="Agent-scoped evaluators are not supported in control templates", - field="condition.evaluator.name", + field=f"{path}.evaluator.name", code="agent_scoped_evaluator_not_supported", message="Agent-scoped evaluators are not supported in control templates.", ) @@ -690,13 +724,13 @@ def _reject_hardcoded_agent_scoped_evaluators( for key in ("and", "or"): children = node.get(key) if isinstance(children, list): - for child in children: + for idx, child in enumerate(children): if isinstance(child, dict): - stack.append(child) + stack.append((child, f"{path}.{key}[{idx}]")) not_child = node.get("not") if isinstance(not_child, dict): - stack.append(not_child) + stack.append((not_child, f"{path}.not")) def render_template_control_input( @@ -707,37 +741,11 @@ def render_template_control_input( """Render a template-backed control input into a concrete control definition.""" template = template_input.template definition_template = template.definition_template - if not isinstance(definition_template, dict): - raise _render_error( - detail="Templates must define a top-level control object", - field="template.definition_template", - code="invalid_definition_template_type", - message="definition_template must be a JSON object representing a control definition.", - ) - for forbidden_key in ("enabled", "name"): - if forbidden_key in definition_template: - raise _render_error( - detail=f"Templates must not define top-level '{forbidden_key}'", - field=forbidden_key, - code="forbidden_template_field", - message=( - f"Templates must not define top-level '{forbidden_key}'. " - "Manage it outside the template." - ), - ) - if "condition" not in definition_template and ( - "selector" in definition_template or "evaluator" in definition_template - ): - raise _render_error( - detail="Templates must use the canonical 'condition' format", - field="condition", - code="legacy_condition_format_not_supported", - message=( - "Templates must use the canonical 'condition' wrapper instead of " - "top-level selector/evaluator fields." - ), - ) + # Reuse structural validation (dict type, forbidden keys, legacy format, + # $param references, unused params, agent-scoped evaluators). + validate_template_structure(template) + assert isinstance(definition_template, dict) # guaranteed by validate_template_structure resolved_values = _resolve_template_values(template, template_input.template_values) reverse_path_map: dict[str, str] = {} diff --git a/server/src/agent_control_server/services/controls.py b/server/src/agent_control_server/services/controls.py index bb8e371a..eba53792 100644 --- a/server/src/agent_control_server/services/controls.py +++ b/server/src/agent_control_server/services/controls.py @@ -3,7 +3,7 @@ from collections.abc import Sequence from dataclasses import dataclass -from agent_control_models import ControlDefinitionRuntime +from agent_control_models import ControlDefinitionRuntime, UnrenderedTemplateControl from agent_control_models.policy import Control as APIControl from sqlalchemy import select, union from sqlalchemy.ext.asyncio import AsyncSession @@ -89,8 +89,6 @@ async def list_controls_for_agent( and c.data.get("template") is not None and c.data.get("condition") is None ): - from agent_control_models import UnrenderedTemplateControl - unrendered = UnrenderedTemplateControl.model_validate(c.data) api_controls.append(APIControl(id=c.id, name=c.name, control=unrendered)) continue diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index b0b5210f..4d4f7d70 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -1658,6 +1658,70 @@ def test_unrendered_template_excluded_from_evaluation(client: TestClient) -> Non assert eval_response.json()["is_safe"] is True +def test_create_unrendered_template_rejects_unknown_value_key( + client: TestClient, +) -> None: + # Given: a template payload with values for a nonexistent parameter + payload = _unrendered_template_payload() + payload["template_values"] = {"nonexistent": "value"} # type: ignore[assignment] + + # When: creating the unrendered control + response = client.put( + "/api/v1/controls", + json={"name": f"unknown-val-{uuid.uuid4()}", "data": payload}, + ) + + # Then: the server rejects with unknown parameter error + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" + assert any( + err.get("code") == "unknown_parameter" + and err.get("field") == "template_values.nonexistent" + for err in body.get("errors", []) + ) + + +def test_create_unrendered_template_rejects_wrong_type_value( + client: TestClient, +) -> None: + # Given: a template with a regex_re2 parameter but we provide a list value + payload = _unrendered_template_payload() + payload["template_values"] = {"pattern": ["not", "a", "string"]} # type: ignore[assignment] + + # When: creating the unrendered control + response = client.put( + "/api/v1/controls", + json={"name": f"wrong-type-{uuid.uuid4()}", "data": payload}, + ) + + # Then: the server rejects with a type error + assert response.status_code == 422 + body = response.json() + assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" + assert any( + err.get("parameter") == "pattern" + for err in body.get("errors", []) + ) + + +def test_unrendered_template_list_shows_template_description_as_fallback( + client: TestClient, +) -> None: + # Given: an unrendered template control whose template has a description + control_id, _ = _create_unrendered_control(client) + + # When: listing controls + response = client.get("/api/v1/controls", params={"template_backed": True}) + + # Then: the summary description falls back to the template description + assert response.status_code == 200, response.text + controls = response.json()["controls"] + unrendered = next((c for c in controls if c["id"] == control_id), None) + assert unrendered is not None + assert unrendered["description"] == "Regex denial template" + + def test_unrendered_template_shows_in_list_with_correct_flags(client: TestClient) -> None: # Given: an unrendered template control control_id, _ = _create_unrendered_control(client) From 045530ade1e83267f42ff778f606889139854cea Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 15:15:44 -0700 Subject: [PATCH 22/25] fix: handle corrupted unrendered template data gracefully - Wrap UnrenderedTemplateControl.model_validate in _parse_stored_control_data with proper error handling (returns 422 CORRUPTED_DATA instead of 500) - Wrap unrendered parse in list_controls_for_agent with try/except to skip corrupted rows instead of crashing the entire listing - Remove redundant unused-parameter check in render_template_control_input (already caught by validate_template_structure called at the top) --- .../endpoints/controls.py | 19 +++++++++++++++- .../services/control_templates.py | 22 +++---------------- .../agent_control_server/services/controls.py | 14 +++++++++++- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 3fb1c0f2..746cd81f 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -109,7 +109,24 @@ def _parse_stored_control_data( ) -> ControlDefinition | UnrenderedTemplateControl: """Parse stored JSONB into the appropriate model type.""" if _is_unrendered_template(data): - return UnrenderedTemplateControl.model_validate(data) + try: + return UnrenderedTemplateControl.model_validate(data) + except ValidationError: + raise APIValidationError( + error_code=ErrorCode.CORRUPTED_DATA, + detail=f"Control '{control_name}' has corrupted unrendered template data", + resource="Control", + resource_id=str(control_id), + hint=f"Update the control data using PUT /api/v1/controls/{control_id}/data.", + errors=[ + ValidationErrorItem( + resource="Control", + field="data", + code="corrupted_data", + message="Stored unrendered template data is invalid.", + ) + ], + ) return parse_control_definition_or_api_error( data, diff --git a/server/src/agent_control_server/services/control_templates.py b/server/src/agent_control_server/services/control_templates.py index 0ab2d9b3..bc800312 100644 --- a/server/src/agent_control_server/services/control_templates.py +++ b/server/src/agent_control_server/services/control_templates.py @@ -759,25 +759,9 @@ def render_template_control_input( template=template, ) - unused_parameters = sorted(set(template.parameters) - referenced_parameters) - if unused_parameters: - raise APIValidationError( - error_code=ErrorCode.TEMPLATE_RENDER_ERROR, - detail="Template defines parameters that are never referenced", - resource="Control", - hint="Remove unused parameters or reference them in definition_template.", - errors=[ - ValidationErrorItem( - resource="Control", - field=f"template.parameters.{parameter_name}", - code="unused_template_parameter", - message=f"Template parameter '{parameter_name}' is never referenced.", - parameter=parameter_name, - parameter_label=template.parameters[parameter_name].label, - ) - for parameter_name in unused_parameters - ], - ) + # Note: unused-parameter detection is handled by validate_template_structure + # (called above). The referenced_parameters set is still tracked here for + # the reverse path map used in error remapping. try: rendered_control = ControlDefinition.model_validate(rendered_payload) diff --git a/server/src/agent_control_server/services/controls.py b/server/src/agent_control_server/services/controls.py index eba53792..15e6d86e 100644 --- a/server/src/agent_control_server/services/controls.py +++ b/server/src/agent_control_server/services/controls.py @@ -8,12 +8,15 @@ from sqlalchemy import select, union from sqlalchemy.ext.asyncio import AsyncSession +from ..logging_utils import get_logger from ..models import Control, agent_controls, agent_policies, policy_controls from .control_definitions import ( parse_control_definition_or_api_error, parse_runtime_control_definition_or_api_error, ) +_logger = get_logger(__name__) + @dataclass(frozen=True) class RuntimeControl: @@ -89,7 +92,16 @@ async def list_controls_for_agent( and c.data.get("template") is not None and c.data.get("condition") is None ): - unrendered = UnrenderedTemplateControl.model_validate(c.data) + try: + unrendered = UnrenderedTemplateControl.model_validate(c.data) + except Exception: + _logger.warning( + "Skipping control '%s' (id=%s): corrupted unrendered template data", + c.name, + c.id, + exc_info=True, + ) + continue api_controls.append(APIControl(id=c.id, name=c.name, control=unrendered)) continue From 52604c8dc9da0be952994e9d9f176ef71462162b Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 15:30:55 -0700 Subject: [PATCH 23/25] fix(sdk): skip unrendered templates in local evaluation and reshape helper - Skip unrendered template controls in check_evaluation_with_local so they don't trigger the server-call fallback (prevents hot-path latency regression for agents with attached unrendered templates) - Accept UnrenderedTemplateControl in to_template_control_input so callers can round-trip unrendered template data from GET endpoints - Add test: unrendered template does not trigger server fallback - Add test: to_template_control_input accepts unrendered template data --- sdks/python/src/agent_control/controls.py | 29 ++++++++-- sdks/python/src/agent_control/evaluation.py | 21 +++++++- sdks/python/tests/test_controls_api.py | 34 ++++++++++++ sdks/python/tests/test_local_evaluation.py | 59 +++++++++++++++++++++ 4 files changed, 137 insertions(+), 6 deletions(-) diff --git a/sdks/python/src/agent_control/controls.py b/sdks/python/src/agent_control/controls.py index a9a19d16..26bc75e1 100644 --- a/sdks/python/src/agent_control/controls.py +++ b/sdks/python/src/agent_control/controls.py @@ -7,6 +7,7 @@ TemplateControlInput, TemplateDefinition, TemplateValue, + UnrenderedTemplateControl, ) from .client import AgentControlClient @@ -266,18 +267,36 @@ async def render_control_template( def to_template_control_input( - data: dict[str, Any] | ControlDefinition, + data: dict[str, Any] | ControlDefinition | UnrenderedTemplateControl, ) -> TemplateControlInput: """Convert stored control data into template authoring input. This is the supported reshape path for template-backed controls returned by ``GET /controls/{id}`` or ``GET /controls/{id}/data`` before submitting them - back to ``set_control_data``. + back to ``set_control_data``. Accepts both rendered (``ControlDefinition``) + and unrendered (``UnrenderedTemplateControl``) shapes. """ + if isinstance(data, UnrenderedTemplateControl): + return TemplateControlInput( + template=data.template, + template_values=dict(data.template_values), + ) if isinstance(data, ControlDefinition): - control_def = data - else: - control_def = ControlDefinition.model_validate(data) + return data.to_template_control_input() + + # Raw dict — detect unrendered vs rendered by checking for condition key. + if ( + isinstance(data, dict) + and data.get("template") is not None + and data.get("condition") is None + ): + unrendered = UnrenderedTemplateControl.model_validate(data) + return TemplateControlInput( + template=unrendered.template, + template_values=dict(unrendered.template_values), + ) + + control_def = ControlDefinition.model_validate(data) return control_def.to_template_control_input() diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index d8df7c79..1c0665f4 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -193,8 +193,18 @@ def _has_applicable_prefiltered_server_controls( parsed_server_controls: list[_ControlAdapter] = [] for control in server_control_payloads: + # Skip unrendered template controls — they have no condition to evaluate + # and should not trigger the server-call fallback. + ctrl_data = control.get("control", {}) + if ( + isinstance(ctrl_data, dict) + and ctrl_data.get("template") is not None + and ctrl_data.get("condition") is None + ): + continue + try: - control_def = ControlDefinitionRuntime.model_validate(control["control"]) + control_def = ControlDefinitionRuntime.model_validate(ctrl_data) parsed_server_controls.append( _ControlAdapter( id=control["id"], @@ -308,6 +318,15 @@ async def check_evaluation_with_local( for control in controls: control_data = control.get("control", {}) + + # Skip unrendered template controls — they cannot be evaluated. + if ( + isinstance(control_data, dict) + and control_data.get("template") is not None + and control_data.get("condition") is None + ): + continue + execution = control_data.get("execution", "server") is_local = execution == "sdk" diff --git a/sdks/python/tests/test_controls_api.py b/sdks/python/tests/test_controls_api.py index 29aa8c02..9e4e36a8 100644 --- a/sdks/python/tests/test_controls_api.py +++ b/sdks/python/tests/test_controls_api.py @@ -271,3 +271,37 @@ def test_to_template_control_input_rejects_raw_control_data() -> None: } ) # Then: the helper rejects the raw control data + + +def test_to_template_control_input_accepts_unrendered_template_data() -> None: + # Given: unrendered template data (template + template_values, no condition) + template_input = agent_control.controls.to_template_control_input( + { + "template": { + "parameters": { + "pattern": { + "type": "regex_re2", + "label": "Pattern", + } + }, + "definition_template": { + "execution": "server", + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {}, + "enabled": False, + } + ) + + # When/Then: the helper extracts template + values successfully + assert isinstance(template_input, TemplateControlInput) + assert template_input.template_values == {} + assert "pattern" in template_input.template.parameters diff --git a/sdks/python/tests/test_local_evaluation.py b/sdks/python/tests/test_local_evaluation.py index 9b56814a..2f33d84f 100644 --- a/sdks/python/tests/test_local_evaluation.py +++ b/sdks/python/tests/test_local_evaluation.py @@ -120,6 +120,37 @@ def add_template_metadata(control: dict[str, Any], *, pattern: str = "test") -> return control +def make_unrendered_template_control( + control_id: int, name: str +) -> dict[str, Any]: + """Build a cached control payload that represents an unrendered template.""" + return { + "id": control_id, + "name": name, + "control": { + "template": { + "parameters": { + "pattern": {"type": "regex_re2", "label": "Pattern"}, + }, + "definition_template": { + "execution": "server", + "scope": {"step_types": ["llm"], "stages": ["pre"]}, + "condition": { + "selector": {"path": "input"}, + "evaluator": { + "name": "regex", + "config": {"pattern": {"$param": "pattern"}}, + }, + }, + "action": {"decision": "deny"}, + }, + }, + "template_values": {}, + "enabled": False, + }, + } + + NON_APPLICABLE_CONTROL_CASES = [ pytest.param({"enabled": False}, id="disabled"), pytest.param({"stage": "post"}, id="stage_mismatch"), @@ -543,6 +574,34 @@ async def test_template_backed_local_control_executes_locally( assert len(result.matches) == 1 assert result.matches[0].control_name == "templated_local_ctrl" + @pytest.mark.asyncio + async def test_unrendered_template_does_not_trigger_server_fallback( + self, + agent_name, + llm_payload, + ): + """Unrendered templates must be skipped, not treated as parse failures.""" + # Given: only an unrendered template control (no rendered controls) + controls = [make_unrendered_template_control(1, "unrendered_ctrl")] + + client = MagicMock(spec=AgentControlClient) + client.http_client = AsyncMock() + client.http_client.post = AsyncMock() + + # When: evaluating with local controls + result = await check_evaluation_with_local( + client=client, + agent_name=agent_name, + step=llm_payload, + stage="pre", + controls=controls, + ) + + # Then: no server call is made (unrendered control skipped, not a fallback trigger) + client.http_client.post.assert_not_called() + # And: result is safe since no evaluable controls exist + assert result.is_safe is True + @pytest.mark.asyncio async def test_local_deny_short_circuits(self, agent_name, llm_payload): """Local deny should return immediately without calling server.""" From 33069ac02589781927929577ee7667121c8377ca Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 15:51:31 -0700 Subject: [PATCH 24/25] fix: align validate endpoint and list filters with unrendered templates - Make /controls/validate mirror create: incomplete template values validate structure only (returns 200) instead of forcing a full render that rejects missing params. Use the render preview endpoint to check renderability. - Exclude unrendered templates from list filters that reference rendered-only fields (execution, step_type, stage, tag). Unrendered templates still appear in unfiltered listings and template_backed filter. - Update validate test to expect 200 for incomplete values - Add test: validate rejects structurally invalid unrendered templates - Add test: unrendered templates excluded from rendered-field filters but included in unfiltered listings --- .../endpoints/controls.py | 17 +++-- server/tests/test_control_templates.py | 66 ++++++++++++++++--- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/server/src/agent_control_server/endpoints/controls.py b/server/src/agent_control_server/endpoints/controls.py index 746cd81f..0412ca63 100644 --- a/server/src/agent_control_server/endpoints/controls.py +++ b/server/src/agent_control_server/endpoints/controls.py @@ -683,12 +683,9 @@ async def validate_control_data( Returns: ValidateControlDataResponse with success=True if valid """ - if isinstance(request.data, TemplateControlInput): - # Validate always attempts full rendering, even if values are incomplete. - # This differs from create/update which allow unrendered storage. - await _render_and_validate_template_input(request.data, db=db, enabled=True) - else: - await _validate_control_definition(request.data, db) + # Validate mirrors create: complete template values trigger a full render, + # incomplete values validate structure only (matching unrendered create). + await _materialize_control_input(request.data, db=db) return ValidateControlDataResponse(success=True) @@ -769,6 +766,12 @@ async def list_controls( else: query = query.where(~Control.data.has_key("template")) + # Filters that reference rendered-only fields exclude unrendered templates + # (which lack condition/execution/scope/tags). + has_rendered_filter = any(f is not None for f in (step_type, stage, execution, tag)) + if has_rendered_filter: + query = query.where(Control.data.has_key("condition")) + if step_type is not None: query = query.where( or_( @@ -817,6 +820,8 @@ async def list_controls( total_query = total_query.where(Control.data.has_key("template")) else: total_query = total_query.where(~Control.data.has_key("template")) + if has_rendered_filter: + total_query = total_query.where(Control.data.has_key("condition")) if step_type is not None: total_query = total_query.where( or_( diff --git a/server/tests/test_control_templates.py b/server/tests/test_control_templates.py index 4d4f7d70..8e08fa6e 100644 --- a/server/tests/test_control_templates.py +++ b/server/tests/test_control_templates.py @@ -995,23 +995,38 @@ def test_template_update_defaults_enabled_to_true_when_stored_key_is_missing( } -def test_template_validate_maps_missing_parameter_error(client: TestClient) -> None: - # Given: a template-backed control payload missing a required parameter value +def test_template_validate_accepts_incomplete_values_as_unrendered( + client: TestClient, +) -> None: + # Given: a template payload with empty values (would be stored as unrendered) + payload = _template_payload() + payload["template_values"] = {} + + # When: validating the payload + response = client.post("/api/v1/controls/validate", json={"data": payload}) + + # Then: validation succeeds (mirrors create behavior for unrendered templates) + assert response.status_code == 200, response.text + assert response.json()["success"] is True + + +def test_template_validate_rejects_structurally_invalid_unrendered( + client: TestClient, +) -> None: + # Given: a template with an undefined $param reference and empty values payload = _template_payload() payload["template_values"] = {} + payload["template"]["definition_template"]["condition"]["evaluator"]["config"]["extra"] = { # type: ignore[index] + "$param": "nonexistent", + } - # When: validating the payload through the public API + # When: validating the payload response = client.post("/api/v1/controls/validate", json={"data": payload}) - # Then: the error is remapped back to the missing template parameter + # Then: structural validation catches the error even without values assert response.status_code == 422 body = response.json() - assert body["error_code"] == "TEMPLATE_PARAMETER_INVALID" - assert any( - err.get("field") == "template_values.pattern" - and err.get("parameter") == "pattern" - for err in body.get("errors", []) - ) + assert body["error_code"] == "TEMPLATE_RENDER_ERROR" def test_template_validate_succeeds_with_defaults_only_payload(client: TestClient) -> None: @@ -1738,6 +1753,37 @@ def test_unrendered_template_shows_in_list_with_correct_flags(client: TestClient assert unrendered["template_rendered"] is False +def test_unrendered_template_excluded_from_rendered_field_filters( + client: TestClient, +) -> None: + # Given: an unrendered template control + control_id, _ = _create_unrendered_control(client) + + # When: filtering by execution (a rendered-only field) + exec_response = client.get("/api/v1/controls", params={"execution": "server"}) + assert exec_response.status_code == 200 + exec_ids = {c["id"] for c in exec_response.json()["controls"]} + + # Then: unrendered template is excluded + assert control_id not in exec_ids + + # When: filtering by step_type + step_response = client.get("/api/v1/controls", params={"step_type": "llm"}) + assert step_response.status_code == 200 + step_ids = {c["id"] for c in step_response.json()["controls"]} + + # Then: unrendered template is excluded + assert control_id not in step_ids + + # When: listing without rendered-field filters + all_response = client.get("/api/v1/controls") + assert all_response.status_code == 200 + all_ids = {c["id"] for c in all_response.json()["controls"]} + + # Then: unrendered template IS included in the unfiltered listing + assert control_id in all_ids + + def test_unrendered_template_can_be_deleted(client: TestClient) -> None: # Given: an unrendered template control control_id, _ = _create_unrendered_control(client) From 1c8628f26eb8c6b2bcc3bfc4e626542e0d71d41c Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 1 Apr 2026 17:41:56 -0700 Subject: [PATCH 25/25] fix: don't count list elements as nesting depth in template validation List elements now inherit their parent's depth instead of incrementing it. A flat array of strings at depth 11 no longer pushes each element to depth 12+, which was incorrectly rejecting real-world templates with nested boolean condition trees containing list-valued parameters. --- models/src/agent_control_models/controls.py | 4 +++- models/tests/test_control_templates.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/models/src/agent_control_models/controls.py b/models/src/agent_control_models/controls.py index f075dc4e..bb27be64 100644 --- a/models/src/agent_control_models/controls.py +++ b/models/src/agent_control_models/controls.py @@ -305,7 +305,9 @@ def _validate_template_definition_structure(value: JsonValue) -> JsonValue: if isinstance(node, dict): stack.extend((nested_value, depth + 1) for nested_value in node.values()) elif isinstance(node, list): - stack.extend((nested_value, depth + 1) for nested_value in node) + # List elements inherit parent depth — a flat array of strings is + # not structurally deeper than a single value. + stack.extend((nested_value, depth) for nested_value in node) return value diff --git a/models/tests/test_control_templates.py b/models/tests/test_control_templates.py index 6217c2c0..893f342a 100644 --- a/models/tests/test_control_templates.py +++ b/models/tests/test_control_templates.py @@ -130,6 +130,27 @@ def test_template_definition_rejects_excessive_nesting() -> None: # Then: the model rejects the deeply nested template +def test_template_definition_allows_flat_arrays_at_depth() -> None: + # Given: a template at nesting depth 11 (just under the limit) that + # contains a flat list of strings — arrays should not count as depth. + deep = _nested_template_value(11) + # Inject a list at the deepest dict level + node = deep + while isinstance(node, dict) and "nested" in node: + if not isinstance(node["nested"], dict): + break + node = node["nested"] + node["items"] = ["a", "b", "c", "d", "e"] + + # When: validating the template definition + result = TemplateDefinition.model_validate( + {"parameters": {}, "definition_template": deep} + ) + + # Then: it succeeds (list elements don't count as additional depth) + assert result.definition_template is not None + + def test_template_definition_rejects_excessive_size() -> None: # Given: a template definition whose structure exceeds the size limit with pytest.raises(