From 94a80acc830b99e7aefbdce9073bef93b51150ee Mon Sep 17 00:00:00 2001 From: ajay-kesavan Date: Tue, 19 May 2026 17:54:53 -0700 Subject: [PATCH] chore(eval): resync evaluator type schemas with Python source Refreshes the 11 evaluator schemas in evaluators_types/ that drifted from their Python config classes since the last regeneration. Pure output of `python -m uipath.eval.evaluators_types.generate_types`, no hand edits. Notable additions surfaced by the regenerator: - Output evaluators (Contains, ExactMatch, JsonSimilarity, LLMJudge*) gain line_by_line_evaluator and line_delimiter, added to OutputEvaluatorConfig in #1481. - All evaluators gain a populated justificationSchema where previously empty. - ToolCall evaluators gain target_output_key documentation. Split out from #1663 (which now ships only the two new classification schemas) so this snapshot refresh can be reviewed independently. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../evaluators_types/ContainsEvaluator.json | 32 ++++++++++++++- .../evaluators_types/ExactMatchEvaluator.json | 32 ++++++++++++++- .../JsonSimilarityEvaluator.json | 40 ++++++++++++++++++- .../LLMJudgeOutputEvaluator.json | 35 +++++++++++++++- ...geStrictJSONSimilarityOutputEvaluator.json | 35 +++++++++++++++- .../LLMJudgeTrajectoryEvaluator.json | 23 ++++++++++- ...LLMJudgeTrajectorySimulationEvaluator.json | 23 ++++++++++- .../ToolCallArgsEvaluator.json | 10 +++++ .../ToolCallCountEvaluator.json | 10 +++++ .../ToolCallOrderEvaluator.json | 22 ++++------ .../ToolCallOutputEvaluator.json | 10 +++++ 11 files changed, 251 insertions(+), 21 deletions(-) diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ContainsEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ContainsEvaluator.json index 9db709f59..a2f41d2c4 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ContainsEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ContainsEvaluator.json @@ -47,6 +47,18 @@ "title": "Target Output Key", "type": "string" }, + "line_by_line_evaluator": { + "default": false, + "description": "If True, split output by delimiter and evaluate each line separately", + "title": "Line By Line Evaluator", + "type": "boolean" + }, + "line_delimiter": { + "default": "\n", + "description": "Delimiter to split output when line_by_line_evaluator is True", + "title": "Line Delimiter", + "type": "string" + }, "case_sensitive": { "default": false, "title": "Case Sensitive", @@ -75,5 +87,23 @@ "title": "ContainsEvaluationCriteria", "type": "object" }, - "justificationSchema": {} + "justificationSchema": { + "description": "Base class for all evaluator justifications.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + } + }, + "required": [ + "expected", + "actual" + ], + "title": "BaseEvaluatorJustification", + "type": "object" + } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json index 866b06416..99478e659 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json @@ -55,6 +55,18 @@ "title": "Target Output Key", "type": "string" }, + "line_by_line_evaluator": { + "default": false, + "description": "If True, split output by delimiter and evaluate each line separately", + "title": "Line By Line Evaluator", + "type": "boolean" + }, + "line_delimiter": { + "default": "\n", + "description": "Delimiter to split output when line_by_line_evaluator is True", + "title": "Line Delimiter", + "type": "string" + }, "case_sensitive": { "default": false, "title": "Case Sensitive", @@ -91,5 +103,23 @@ "title": "OutputEvaluationCriteria", "type": "object" }, - "justificationSchema": {} + "justificationSchema": { + "description": "Base class for all evaluator justifications.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + } + }, + "required": [ + "expected", + "actual" + ], + "title": "BaseEvaluatorJustification", + "type": "object" + } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/evaluators_types/JsonSimilarityEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/JsonSimilarityEvaluator.json index ef17bf083..5e40ee0b0 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/JsonSimilarityEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/JsonSimilarityEvaluator.json @@ -54,6 +54,18 @@ "description": "Key to extract output from agent execution", "title": "Target Output Key", "type": "string" + }, + "line_by_line_evaluator": { + "default": false, + "description": "If True, split output by delimiter and evaluate each line separately", + "title": "Line By Line Evaluator", + "type": "boolean" + }, + "line_delimiter": { + "default": "\n", + "description": "Delimiter to split output when line_by_line_evaluator is True", + "title": "Line Delimiter", + "type": "string" } }, "title": "JsonSimilarityEvaluatorConfig", @@ -82,6 +94,32 @@ "type": "object" }, "justificationSchema": { - "type": "string" + "description": "Justification for the JSON similarity evaluator.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, + "matched_leaves": { + "title": "Matched Leaves", + "type": "number" + }, + "total_leaves": { + "title": "Total Leaves", + "type": "number" + } + }, + "required": [ + "expected", + "actual", + "matched_leaves", + "total_leaves" + ], + "title": "JsonSimilarityJustification", + "type": "object" } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeOutputEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeOutputEvaluator.json index 06f731c1f..642ea0a50 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeOutputEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeOutputEvaluator.json @@ -81,6 +81,18 @@ "description": "Key to extract output from agent execution", "title": "Target Output Key", "type": "string" + }, + "line_by_line_evaluator": { + "default": false, + "description": "If True, split output by delimiter and evaluate each line separately", + "title": "Line By Line Evaluator", + "type": "boolean" + }, + "line_delimiter": { + "default": "\n", + "description": "Delimiter to split output when line_by_line_evaluator is True", + "title": "Line Delimiter", + "type": "string" } }, "title": "LLMJudgeOutputEvaluatorConfig", @@ -109,6 +121,27 @@ "type": "object" }, "justificationSchema": { - "type": "string" + "description": "Justification for LLM judge evaluators.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, + "justification": { + "title": "Justification", + "type": "string" + } + }, + "required": [ + "expected", + "actual", + "justification" + ], + "title": "LLMJudgeJustification", + "type": "object" } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeStrictJSONSimilarityOutputEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeStrictJSONSimilarityOutputEvaluator.json index 0fffbbe81..e1e154244 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeStrictJSONSimilarityOutputEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeStrictJSONSimilarityOutputEvaluator.json @@ -81,6 +81,18 @@ "description": "Key to extract output from agent execution", "title": "Target Output Key", "type": "string" + }, + "line_by_line_evaluator": { + "default": false, + "description": "If True, split output by delimiter and evaluate each line separately", + "title": "Line By Line Evaluator", + "type": "boolean" + }, + "line_delimiter": { + "default": "\n", + "description": "Delimiter to split output when line_by_line_evaluator is True", + "title": "Line Delimiter", + "type": "string" } }, "title": "LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig", @@ -109,6 +121,27 @@ "type": "object" }, "justificationSchema": { - "type": "string" + "description": "Justification for LLM judge evaluators.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, + "justification": { + "title": "Justification", + "type": "string" + } + }, + "required": [ + "expected", + "actual", + "justification" + ], + "title": "LLMJudgeJustification", + "type": "object" } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectoryEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectoryEvaluator.json index 8695fb738..fdaa6e5cb 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectoryEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectoryEvaluator.json @@ -87,6 +87,27 @@ "type": "object" }, "justificationSchema": { - "type": "string" + "description": "Justification for LLM judge evaluators.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, + "justification": { + "title": "Justification", + "type": "string" + } + }, + "required": [ + "expected", + "actual", + "justification" + ], + "title": "LLMJudgeJustification", + "type": "object" } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectorySimulationEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectorySimulationEvaluator.json index 006e24202..78ec51450 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectorySimulationEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectorySimulationEvaluator.json @@ -87,6 +87,27 @@ "type": "object" }, "justificationSchema": { - "type": "string" + "description": "Justification for LLM judge evaluators.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, + "justification": { + "title": "Justification", + "type": "string" + } + }, + "required": [ + "expected", + "actual", + "justification" + ], + "title": "LLMJudgeJustification", + "type": "object" } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallArgsEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallArgsEvaluator.json index 645ada479..22d3447e2 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallArgsEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallArgsEvaluator.json @@ -120,6 +120,14 @@ "justificationSchema": { "description": "Justification for the tool call args evaluator.", "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, "explained_tool_calls_args": { "additionalProperties": { "type": "string" @@ -129,6 +137,8 @@ } }, "required": [ + "expected", + "actual", "explained_tool_calls_args" ], "title": "ToolCallArgsEvaluatorJustification", diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallCountEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallCountEvaluator.json index 56b56d543..3f9b30e02 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallCountEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallCountEvaluator.json @@ -93,6 +93,14 @@ "justificationSchema": { "description": "Justification for the tool call count evaluator.", "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, "explained_tool_calls_count": { "additionalProperties": { "type": "string" @@ -102,6 +110,8 @@ } }, "required": [ + "expected", + "actual", "explained_tool_calls_count" ], "title": "ToolCallCountEvaluatorJustification", diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOrderEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOrderEvaluator.json index 568890eb1..79cd0df58 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOrderEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOrderEvaluator.json @@ -73,19 +73,13 @@ "justificationSchema": { "description": "Justification for the tool call order evaluator.", "properties": { - "actual_tool_calls_order": { - "items": { - "type": "string" - }, - "title": "Actual Tool Calls Order", - "type": "array" + "expected": { + "title": "Expected", + "type": "string" }, - "expected_tool_calls_order": { - "items": { - "type": "string" - }, - "title": "Expected Tool Calls Order", - "type": "array" + "actual": { + "title": "Actual", + "type": "string" }, "lcs": { "items": { @@ -96,8 +90,8 @@ } }, "required": [ - "actual_tool_calls_order", - "expected_tool_calls_order", + "expected", + "actual", "lcs" ], "title": "ToolCallOrderEvaluatorJustification", diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOutputEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOutputEvaluator.json index 73455592a..4c1bf029d 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOutputEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ToolCallOutputEvaluator.json @@ -113,6 +113,14 @@ "justificationSchema": { "description": "Justification for the tool call output evaluator.", "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + }, "explained_tool_calls_outputs": { "additionalProperties": { "type": "string" @@ -122,6 +130,8 @@ } }, "required": [ + "expected", + "actual", "explained_tool_calls_outputs" ], "title": "ToolCallOutputEvaluatorJustification",