feat(eval): classification evaluator schemas + sample projects + e2e tests

packages/uipath/samples/binary_classification_agent/bindings.json

@@ -0,0 +1,4 @@
+{
+  "version": "2.0",
+  "resources": []
+}

packages/uipath/samples/binary_classification_agent/evaluations/eval-sets/default.json

@@ -0,0 +1,63 @@
+{
+  "version": "1.0",
+  "id": "SpamBinaryEval",
+  "name": "Binary spam classifier — precision",
+  "evaluatorRefs": ["BinarySpamPrecision"],
+  "evaluations": [
+    {
+      "id": "spam-prize",
+      "name": "Spam: prize giveaway",
+      "inputs": {
+        "email_subject": "You won a FREE iPhone!!!",
+        "email_body": "Congratulations! Click here to claim your prize now."
+      },
+      "evaluationCriterias": {
+        "BinarySpamPrecision": { "expectedClass": "spam" }
+      }
+    },
+    {
+      "id": "spam-promo",
+      "name": "Spam: unsolicited promo",
+      "inputs": {
+        "email_subject": "Winner of the monthly drawing",
+        "email_body": "You've been selected. Click here to redeem."
+      },
+      "evaluationCriterias": {
+        "BinarySpamPrecision": { "expectedClass": "spam" }
+      }
+    },
+    {
+      "id": "ham-invoice",
+      "name": "Ham: legitimate invoice",
+      "inputs": {
+        "email_subject": "Your March invoice is ready",
+        "email_body": "Your monthly invoice of $45.99 is attached. Payment is due March 15."
+      },
+      "evaluationCriterias": {
+        "BinarySpamPrecision": { "expectedClass": "ham" }
+      }
+    },
+    {
+      "id": "ham-meeting",
+      "name": "Ham: meeting request",
+      "inputs": {
+        "email_subject": "Sync on Q2 planning",
+        "email_body": "Can we meet Wednesday at 2pm to align on next quarter's roadmap?"
+      },
+      "evaluationCriterias": {
+        "BinarySpamPrecision": { "expectedClass": "ham" }
+      }
+    },
+    {
+      "id": "ham-mislabeled",
+      "name": "Ham mislabeled as spam (forces a false positive)",
+      "inputs": {
+        "email_subject": "Free coffee in the break room!!!",
+        "email_body": "Just a heads up — the new espresso machine is set up."
+      },
+      "evaluationCriterias": {
+        "BinarySpamPrecision": { "expectedClass": "ham" }
+      }
+    }
+  ]
+}

packages/uipath/samples/binary_classification_agent/evaluations/evaluators/binary-classification.json

@@ -0,0 +1,16 @@
+{
+  "version": "1.0",
+  "id": "BinarySpamPrecision",
+  "description": "Precision on the 'spam' positive class",
+  "evaluatorTypeId": "uipath-binary-classification",
+  "evaluatorConfig": {
+    "name": "BinarySpamPrecision",
+    "targetOutputKey": "category",
+    "positiveClass": "spam",
+    "metricType": "precision",
+    "fValue": 1.0,
+    "defaultEvaluationCriteria": {
+      "expectedClass": "ham"
+    }
+  }
+}

packages/uipath/samples/binary_classification_agent/main.py

@@ -0,0 +1,39 @@
+"""Rule-based spam/ham classifier demonstrating the binary classification evaluator."""
+
+from dataclasses import dataclass
+
+from uipath.tracing import traced
+
+SPAMMY_TOKENS = {
+    "free",
+    "winner",
+    "congratulations",
+    "click here",
+    "prize",
+    "!!!",
+}
+
+
+@dataclass
+class EmailInput:
+    email_subject: str
+    email_body: str
+
+
+@dataclass
+class Classification:
+    category: str
+
+
+@traced(name="classify_email", span_type="tool")
+def classify_email(subject: str, body: str) -> str:
+    """Return 'spam' if any spam-indicator token appears in the subject or body."""
+    text = f"{subject} {body}".lower()
+    return "spam" if any(token in text for token in SPAMMY_TOKENS) else "ham"
+
+
+@traced()
+async def main(input: EmailInput) -> Classification:
+    """Classify an email as 'spam' or 'ham'."""
+    category = classify_email(input.email_subject, input.email_body)
+    return Classification(category=category)

packages/uipath/samples/binary_classification_agent/pyproject.toml

@@ -0,0 +1,9 @@
+[project]
+name = "binary-classification-agent"
+version = "0.0.1"
+description = "Rule-based spam/ham classifier demonstrating the binary classification evaluator"
+requires-python = ">=3.11"
+dependencies = ["uipath"]
+
+[dependency-groups]
+dev = ["uipath-dev"]

packages/uipath/samples/binary_classification_agent/uipath.json

@@ -0,0 +1,5 @@
+{
+  "functions": {
+    "main": "main.py:main"
+  }
+}

packages/uipath/samples/multiclass_classification_simple/bindings.json

@@ -0,0 +1,4 @@
+{
+  "version": "2.0",
+  "resources": []
+}

packages/uipath/samples/multiclass_classification_simple/evaluations/eval-sets/default.json

@@ -0,0 +1,85 @@
+{
+  "version": "1.0",
+  "id": "EmailMulticlassEval",
+  "name": "3-class email router — macro F1",
+  "evaluatorRefs": ["EmailMulticlassFScore"],
+  "evaluations": [
+    {
+      "id": "pay-invoice",
+      "name": "Payments: invoice reminder",
+      "inputs": {
+        "email_subject": "Your March invoice is ready",
+        "email_body": "Your monthly invoice of $45.99 is now available. Payment is due March 15."
+      },
+      "evaluationCriterias": {
+        "EmailMulticlassFScore": { "expectedClass": "payments" }
+      }
+    },
+    {
+      "id": "pay-refund",
+      "name": "Payments: refund request",
+      "inputs": {
+        "email_subject": "Refund for last month's charge",
+        "email_body": "I was charged twice for the same service. Please process a refund."
+      },
+      "evaluationCriterias": {
+        "EmailMulticlassFScore": { "expectedClass": "payments" }
+      }
+    },
+    {
+      "id": "support-broken",
+      "name": "Support: feature broken",
+      "inputs": {
+        "email_subject": "Login is broken",
+        "email_body": "I'm getting an error when trying to sign in. Need help."
+      },
+      "evaluationCriterias": {
+        "EmailMulticlassFScore": { "expectedClass": "support" }
+      }
+    },
+    {
+      "id": "support-question",
+      "name": "Support: how-to question",
+      "inputs": {
+        "email_subject": "How do I export my data?",
+        "email_body": "Can you help me figure out where the export button is?"
+      },
+      "evaluationCriterias": {
+        "EmailMulticlassFScore": { "expectedClass": "support" }
+      }
+    },
+    {
+      "id": "spam-prize",
+      "name": "Spam: prize giveaway",
+      "inputs": {
+        "email_subject": "You won a FREE iPhone!!!",
+        "email_body": "Congratulations! Click here to claim your prize."
+      },
+      "evaluationCriterias": {
+        "EmailMulticlassFScore": { "expectedClass": "spam" }
+      }
+    },
+    {
+      "id": "spam-promo",
+      "name": "Spam: marketing winner",
+      "inputs": {
+        "email_subject": "Winner of the monthly drawing",
+        "email_body": "Congratulations, click here to redeem your reward."
+      },
+      "evaluationCriterias": {
+        "EmailMulticlassFScore": { "expectedClass": "spam" }
+      }
+    },
+    {
+      "id": "support-misrouted-by-payment-word",
+      "name": "Support email accidentally routed to payments (forces an FP for payments)",
+      "inputs": {
+        "email_subject": "Question about my billing portal access",
+        "email_body": "I cannot log into the billing portal. The page just spins. Can you help?"
+      },
+      "evaluationCriterias": {
+        "EmailMulticlassFScore": { "expectedClass": "support" }
+      }
+    }
+  ]
+}

packages/uipath/samples/multiclass_classification_simple/evaluations/evaluators/multiclass-classification.json

@@ -0,0 +1,17 @@
+{
+  "version": "1.0",
+  "id": "EmailMulticlassFScore",
+  "description": "Macro-averaged F1 across payments / support / spam",
+  "evaluatorTypeId": "uipath-multiclass-classification",
+  "evaluatorConfig": {
+    "name": "EmailMulticlassFScore",
+    "targetOutputKey": "category",
+    "classes": ["payments", "support", "spam"],
+    "metricType": "f-score",
+    "averaging": "macro",
+    "fValue": 1.0,
+    "defaultEvaluationCriteria": {
+      "expectedClass": "support"
+    }
+  }
+}

packages/uipath/samples/multiclass_classification_simple/main.py

@@ -0,0 +1,51 @@
+"""Rule-based 3-class email router demonstrating the multiclass classification evaluator."""
+
+from dataclasses import dataclass
+
+from uipath.tracing import traced
+
+SPAM_TOKENS = {"free", "winner", "congratulations", "click here", "prize", "!!!"}
+PAYMENT_TOKENS = {"invoice", "payment", "refund", "charge", "billing", "$"}
+SUPPORT_TOKENS = {
+    "help",
+    "support",
+    "issue",
+    "error",
+    "ticket",
+    "broken",
+    "not working",
+}
+
+
+@dataclass
+class EmailInput:
+    email_subject: str
+    email_body: str
+
+
+@dataclass
+class Classification:
+    category: str
+
+
+@traced(name="classify_email", span_type="tool")
+def classify_email(subject: str, body: str) -> str:
+    """Classify into 'spam', 'payments', or 'support' using priority rules.
+
+    Spam is checked first so promos with billing-flavored words still route to spam.
+    Payments is checked before support because it is the more specific intent.
+    Support is the catch-all default.
+    """
+    text = f"{subject} {body}".lower()
+    if any(token in text for token in SPAM_TOKENS):
+        return "spam"
+    if any(token in text for token in PAYMENT_TOKENS):
+        return "payments"
+    return "support"
+
+
+@traced()
+async def main(input: EmailInput) -> Classification:
+    """Route an email to one of three queues."""
+    category = classify_email(input.email_subject, input.email_body)
+    return Classification(category=category)

packages/uipath/samples/multiclass_classification_simple/pyproject.toml

@@ -0,0 +1,9 @@
+[project]
+name = "multiclass-classification-simple"
+version = "0.0.1"
+description = "Rule-based 3-class email router demonstrating the multiclass classification evaluator with macro-averaged F1"
+requires-python = ">=3.11"
+dependencies = ["uipath"]
+
+[dependency-groups]
+dev = ["uipath-dev"]

packages/uipath/samples/multiclass_classification_simple/uipath.json

@@ -0,0 +1,5 @@
+{
+  "functions": {
+    "main": "main.py:main"
+  }
+}