diff --git a/benchmarks/ablation_eu_ai_act.py b/benchmarks/ablation_eu_ai_act.py new file mode 100644 index 0000000..b8f6c9b --- /dev/null +++ b/benchmarks/ablation_eu_ai_act.py @@ -0,0 +1,191 @@ +"""4-way ablation on the eu-ai-act-prohibited pack. + +Compares macro F1 / R / P / FP rate at the pack's default threshold (1.5) +across four configurations: + A = baseline (no lexical_groups, no policy_overrides) + B = +lexical only (lexical_groups on, policy_overrides off) + C = +policy only (lexical_groups off, policy_overrides on) + D = +both (lexical_groups on, policy_overrides on) + +For each config we mutate a temp copy of the pack's _ns.json, load it as +a microresolve namespace, run the 100 prohibited + 80 benign corpus +through it, and compute per-intent + macro metrics. + +Pass criteria (locked before measurement): + - D ≥ A + 1pp F1 + - D ≥ B + 0.5pp F1 (proves policy adds value over morph) + - No pack regresses >1pp F1 from B → D + - Benign FP rate increases ≤2pp from B → D anywhere +""" +import json +import shutil +import sys +from pathlib import Path +from collections import defaultdict + +import microresolve + +PACK_NAME = "eu-ai-act-prohibited" +PACK_SRC = Path("packs") / PACK_NAME +CORPUS = Path("_internal/EU_AI_ACT_EVAL_CORPUS.json") +TARGET_THRESHOLD = 1.5 +GAP = 1.5 + + +def stage_pack(config: str, root: Path) -> Path: + """Stage the pack to //, mutating _ns.json per config.""" + cfg_root = root / config + if cfg_root.exists(): + shutil.rmtree(cfg_root) + cfg_root.mkdir(parents=True) + dest = cfg_root / PACK_NAME + shutil.copytree(PACK_SRC, dest) + + ns_path = dest / "_ns.json" + ns = json.load(open(ns_path)) + if config in ("baseline", "policy_only"): + ns.pop("lexical_groups", None) + if config in ("baseline", "lex_only"): + ns.pop("policy_overrides", None) + json.dump(ns, open(ns_path, "w"), indent=2) + return cfg_root + + +def run_config(config: str, root: Path, corpus: dict) -> dict: + cfg_root = stage_pack(config, root) + engine = microresolve.MicroResolve(data_dir=str(cfg_root)) + ns = engine.namespace(PACK_NAME) + + # Resolve every query, score it against ground truth + per_intent = defaultdict(lambda: {"tp": 0, "fn": 0, "fp": 0, "tn": 0}) + intent_ids = ns.intent_ids() + benign_hits = 0 + benign_total = 0 + + for entry in corpus["prohibited"]: + gt = entry["expected_intent"] + query = entry["text"] + result = ns.resolve(query) + hit_high = any( + i.band == "High" and i.score >= TARGET_THRESHOLD for i in result.intents + ) + top = next( + (i for i in result.intents if i.score >= TARGET_THRESHOLD), + None, + ) + for iid in intent_ids: + is_gt = iid == gt + is_hit = top is not None and top.id == iid + if is_gt and is_hit: + per_intent[iid]["tp"] += 1 + elif is_gt and not is_hit: + per_intent[iid]["fn"] += 1 + elif not is_gt and is_hit: + per_intent[iid]["fp"] += 1 + else: + per_intent[iid]["tn"] += 1 + + for entry in corpus["benign"]: + query = entry["text"] + result = ns.resolve(query) + benign_total += 1 + top = next( + (i for i in result.intents if i.score >= TARGET_THRESHOLD), + None, + ) + if top is not None and top.id != "legitimate_use": + benign_hits += 1 + for iid in intent_ids: + if iid == top.id and iid != "legitimate_use": + per_intent[iid]["fp"] += 1 + + # Compute macros + metrics = {} + p_sum = r_sum = f_sum = 0.0 + n = 0 + for iid in intent_ids: + if iid == "legitimate_use": + continue + m = per_intent[iid] + tp, fn, fp = m["tp"], m["fn"], m["fp"] + p = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + r = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + f = 2 * p * r / (p + r) if (p + r) > 0 else 0.0 + metrics[iid] = {"p": p, "r": r, "f1": f, "tp": tp, "fn": fn, "fp": fp} + p_sum += p + r_sum += r + f_sum += f + n += 1 + + benign_fp_rate = benign_hits / benign_total if benign_total > 0 else 0.0 + + return { + "config": config, + "macro_p": p_sum / n, + "macro_r": r_sum / n, + "macro_f1": f_sum / n, + "benign_fp_rate": benign_fp_rate, + "benign_hits": benign_hits, + "benign_total": benign_total, + "per_intent": metrics, + } + + +def main(): + corpus = json.load(open(CORPUS)) + root = Path("/tmp/ablation") + if root.exists(): + shutil.rmtree(root) + root.mkdir(parents=True) + + configs = ["baseline", "lex_only", "policy_only", "both"] + results = {} + for c in configs: + print(f"--- {c} ---", flush=True) + results[c] = run_config(c, root, corpus) + r = results[c] + print( + f" macro: P={r['macro_p']:.3f} R={r['macro_r']:.3f} F1={r['macro_f1']:.3f}" + f" benign-FP={r['benign_fp_rate']:.3f} ({r['benign_hits']}/{r['benign_total']})" + ) + + out = Path("benchmarks/results/ablation_eu_ai_act.json") + out.parent.mkdir(exist_ok=True) + json.dump(results, open(out, "w"), indent=2) + + print() + print("=" * 72) + print("Summary config F1 ΔF1 R ΔR P benign-FP") + print("=" * 72) + base = results["baseline"] + for c in configs: + r = results[c] + d_f1 = (r["macro_f1"] - base["macro_f1"]) * 100 + d_r = (r["macro_r"] - base["macro_r"]) * 100 + print( + f" {c:13s} {r['macro_f1']:.3f} {d_f1:+5.1f}pp {r['macro_r']:.3f} {d_r:+5.1f}pp {r['macro_p']:.3f} {r['benign_fp_rate']:.3f}" + ) + + print() + print("Pass criteria check:") + b = results["baseline"] + lex = results["lex_only"] + both = results["both"] + crit1 = (both["macro_f1"] - b["macro_f1"]) >= 0.01 + crit2 = (both["macro_f1"] - lex["macro_f1"]) >= 0.005 + crit3 = (lex["macro_f1"] - both["macro_f1"]) <= 0.01 + crit4 = (both["benign_fp_rate"] - lex["benign_fp_rate"]) <= 0.02 + print(f" D > A + 1pp F1? {crit1} ({(both['macro_f1']-b['macro_f1'])*100:+.2f}pp)") + print(f" D > B + 0.5pp F1? {crit2} ({(both['macro_f1']-lex['macro_f1'])*100:+.2f}pp)") + print(f" D - B regression ≤ 1pp F1? {crit3}") + print(f" D - B benign-FP ≤ 2pp? {crit4} ({(both['benign_fp_rate']-lex['benign_fp_rate'])*100:+.2f}pp)") + all_pass = crit1 and crit2 and crit3 and crit4 + print(f"\n OVERALL: {'PASS — ship combined' if all_pass else 'KILL — strip policy_overrides'}") + + print() + print(f"Full results written to {out}") + return 0 if all_pass else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/bin/server/main.rs b/src/bin/server/main.rs index a335bef..2ebaee2 100644 --- a/src/bin/server/main.rs +++ b/src/bin/server/main.rs @@ -24,6 +24,7 @@ mod routes_intents; mod routes_lexical; mod routes_logs; mod routes_phrases; +mod routes_policy_overrides; mod routes_projects; mod routes_review; mod routes_settings; @@ -378,6 +379,7 @@ async fn main() { // here. let protected_api = axum::Router::new() .merge(routes_core::routes()) + .merge(routes_policy_overrides::routes()) .merge(routes_intents::routes()) .merge(routes_lexical::routes()) .merge(routes_logs::routes()) diff --git a/src/bin/server/routes_core.rs b/src/bin/server/routes_core.rs index e6860b9..3e6e6e4 100644 --- a/src/bin/server/routes_core.rs +++ b/src/bin/server/routes_core.rs @@ -97,7 +97,17 @@ pub async fn resolve( // Audit: record the no-match decision too (compliance buyers // need to see "the system saw this query and declined to fire"). - audit_resolve(&state, &kid, &app_id, &req.query, &[], 0.0, latency_us); + let no_match_trace = opt_trace.as_ref().map(build_compact_audit_trace); + audit_resolve( + &state, + &kid, + &app_id, + &req.query, + &[], + 0.0, + latency_us, + no_match_trace, + ); let trace_val = opt_trace.map(|t| build_trace_json(&t)); let mut resp = serde_json::json!({ @@ -202,8 +212,20 @@ pub async fn resolve( } // ── Audit log: tamper-evident decision record ──────────────────── + // When the caller asked for a trace, embed a compact summary in the + // audit payload too — this is what makes Art. 13 interpretive + // transparency real (you can defend not just "we routed" but "we + // routed because tokens X, Y, Z"). + let compact_trace = opt_trace.as_ref().map(build_compact_audit_trace); audit_resolve( - &state, &kid, &app_id, &req.query, &intents, threshold, latency_us, + &state, + &kid, + &app_id, + &req.query, + &intents, + threshold, + latency_us, + compact_trace, ); let mut resp = serde_json::json!({ @@ -224,7 +246,11 @@ pub async fn resolve( /// shapes the payload and serializes the chain write. The query is /// stored as a SHA-256 hash (PII-friendly) — auditors can verify /// "decision X happened for query Y" by hashing Y and looking it up, -/// without the operator retaining raw queries. +/// without the operator retaining raw queries. When `compact_trace` +/// is supplied, it lands inside the payload — surfaces *why* a routing +/// happened, not just *that* it happened (Art. 13 interpretive +/// transparency in the audit chain). +#[allow(clippy::too_many_arguments)] fn audit_resolve( state: &AppState, kid: &str, @@ -233,17 +259,21 @@ fn audit_resolve( intents: &[serde_json::Value], threshold: f32, latency_us: u64, + compact_trace: Option, ) { if !state.audit_log.mode().enabled() { return; } - let payload = serde_json::json!({ + let mut payload = serde_json::json!({ "ns": app_id, "query_hash": hash_query(query), "intents": intents, "threshold_applied": threshold, "latency_us": latency_us, }); + if let Some(t) = compact_trace { + payload["trace"] = t; + } state.audit_log.record(kid, app_id, "resolve", payload); } @@ -259,5 +289,67 @@ fn build_trace_json(t: µresolve::ResolveTrace) -> serde_json::Value { }, "negated": t.negated, "threshold_applied": t.threshold_applied, + "per_token": t.per_token.iter().map(|c| serde_json::json!({ + "token": c.token, + "intent": c.intent, + "weight": (c.weight * 1000.0).round() / 1000.0, + "idf": (c.idf * 1000.0).round() / 1000.0, + "delta": (c.delta * 1000.0).round() / 1000.0, + "negated": c.negated, + })).collect::>(), + "per_intent": t.per_intent.iter().map(|s| serde_json::json!({ + "intent": s.intent, + "raw_score": (s.raw_score * 100.0).round() / 100.0, + "voting_tokens": s.voting_tokens, + "voting_multiplier": (s.voting_multiplier * 100.0).round() / 100.0, + "policy_overrides_bonus": (s.policy_overrides_bonus * 100.0).round() / 100.0, + "policy_overrides_fired": s.policy_overrides_fired, + })).collect::>(), + "explanation": t.explanation, + }) +} + +/// Compact trace summary for audit log entries: top intents (with voting state +/// and any conjunctions that fired) and top 5 token contributions. Designed +/// to be small enough to live inside every resolve audit event without +/// bloating the chain. Full trace stays in the API response only when requested. +fn build_compact_audit_trace(t: µresolve::ResolveTrace) -> serde_json::Value { + let top_intents: Vec = t + .per_intent + .iter() + .take(3) + .map(|s| { + serde_json::json!({ + "intent": s.intent, + "raw_score": (s.raw_score * 100.0).round() / 100.0, + "voting_tokens": s.voting_tokens, + "policy_overrides_fired": s.policy_overrides_fired, + }) + }) + .collect(); + // Top 5 by absolute delta, picking the highest-impact contributions only. + let mut sorted_contrib: Vec<µresolve::scoring::TokenContribution> = + t.per_token.iter().collect(); + sorted_contrib.sort_by(|a, b| { + b.delta + .abs() + .partial_cmp(&a.delta.abs()) + .unwrap_or(std::cmp::Ordering::Equal) + }); + let top_tokens: Vec = sorted_contrib + .iter() + .take(5) + .map(|c| { + serde_json::json!({ + "token": c.token, + "intent": c.intent, + "delta": (c.delta * 1000.0).round() / 1000.0, + }) + }) + .collect(); + serde_json::json!({ + "top_intents": top_intents, + "top_tokens": top_tokens, + "explanation": t.explanation, }) } diff --git a/src/bin/server/routes_policy_overrides.rs b/src/bin/server/routes_policy_overrides.rs new file mode 100644 index 0000000..8ecc2da --- /dev/null +++ b/src/bin/server/routes_policy_overrides.rs @@ -0,0 +1,169 @@ +//! Policy override CRUD — narrow declarative escape hatch per namespace. +//! +//! A policy override is a hard rule: when ALL listed words appear in the +//! normalised query, add `bonus` to the target intent. Designed for +//! pack-author encoded policy assertions that are too important or too rare +//! to wait for the auto-learn loop to teach them — explicit Article 5 +//! carve-outs, CSAM detection vs generation, similar externally-specified +//! distinctions. Usage guideline: ≤10 per pack. Most compositional logic +//! belongs in per-intent structured expansion (LLM-distilled at pack-build), +//! not here. +//! +//! Internally still backed by `PolicyOverride` — the runtime mechanism is a +//! token-conjunction. The renamed surface signals the intended *role*: hard +//! policy override, not a Swiss Army knife. +//! +//! Every mutation lands in the audit log so operators can see who added / +//! removed which rule when. + +use crate::state::*; +use axum::{ + extract::{Path, State}, + http::{HeaderMap, StatusCode}, + routing::{delete, get}, + Extension, Json, +}; + +pub fn routes() -> axum::Router { + axum::Router::new() + .route("/api/policy-overrides", get(list).post(add)) + .route("/api/policy-overrides/{idx}", delete(remove).patch(update)) +} + +#[derive(serde::Deserialize)] +pub struct PolicyOverridePayload { + pub words: Vec, + pub intent: String, + pub bonus: f32, +} + +fn rule_to_json(idx: usize, r: µresolve::scoring::PolicyOverride) -> serde_json::Value { + serde_json::json!({ + "idx": idx, + "words": r.words, + "intent": r.intent, + "bonus": r.bonus, + }) +} + +pub async fn list( + State(state): State, + headers: HeaderMap, +) -> Result, (StatusCode, String)> { + let app_id = app_id_from_headers(&headers); + let h = state.engine.try_namespace(&app_id).ok_or(( + StatusCode::NOT_FOUND, + format!("namespace '{}' not found", app_id), + ))?; + let rules = h.list_policy_overrides(); + let arr: Vec = rules + .iter() + .enumerate() + .map(|(i, r)| rule_to_json(i, r)) + .collect(); + Ok(Json(serde_json::json!({ "policy_overrides": arr }))) +} + +pub async fn add( + State(state): State, + headers: HeaderMap, + Extension(KeyName(kid)): Extension, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let app_id = app_id_from_headers(&headers); + let h = state.engine.try_namespace(&app_id).ok_or(( + StatusCode::NOT_FOUND, + format!("namespace '{}' not found", app_id), + ))?; + + let words_for_audit = req.words.clone(); + let intent_for_audit = req.intent.clone(); + let bonus_for_audit = req.bonus; + + let idx = h + .add_policy_override(req.words, req.intent, req.bonus) + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; + + audit_mutation( + &state, + &kid, + &app_id, + "policy_override.add", + serde_json::json!({ + "idx": idx, + "words": words_for_audit, + "intent": intent_for_audit, + "bonus": bonus_for_audit, + }), + ); + + let _ = h.flush(); + maybe_commit(&state, &app_id); + + Ok(Json(serde_json::json!({ "idx": idx }))) +} + +pub async fn remove( + State(state): State, + headers: HeaderMap, + Extension(KeyName(kid)): Extension, + Path(idx): Path, +) -> Result { + let app_id = app_id_from_headers(&headers); + let h = state.engine.try_namespace(&app_id).ok_or(( + StatusCode::NOT_FOUND, + format!("namespace '{}' not found", app_id), + ))?; + let removed = h + .remove_policy_override(idx) + .map_err(|e| (StatusCode::NOT_FOUND, e.to_string()))?; + audit_mutation( + &state, + &kid, + &app_id, + "policy_override.remove", + serde_json::json!({ + "idx": idx, + "words": removed.words, + "intent": removed.intent, + "bonus": removed.bonus, + }), + ); + let _ = h.flush(); + maybe_commit(&state, &app_id); + Ok(StatusCode::NO_CONTENT) +} + +pub async fn update( + State(state): State, + headers: HeaderMap, + Extension(KeyName(kid)): Extension, + Path(idx): Path, + Json(req): Json, +) -> Result { + let app_id = app_id_from_headers(&headers); + let h = state.engine.try_namespace(&app_id).ok_or(( + StatusCode::NOT_FOUND, + format!("namespace '{}' not found", app_id), + ))?; + let words_for_audit = req.words.clone(); + let intent_for_audit = req.intent.clone(); + let bonus_for_audit = req.bonus; + h.update_policy_override(idx, req.words, req.intent, req.bonus) + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; + audit_mutation( + &state, + &kid, + &app_id, + "policy_override.update", + serde_json::json!({ + "idx": idx, + "words": words_for_audit, + "intent": intent_for_audit, + "bonus": bonus_for_audit, + }), + ); + let _ = h.flush(); + maybe_commit(&state, &app_id); + Ok(StatusCode::NO_CONTENT) +} diff --git a/src/engine.rs b/src/engine.rs index 975435b..2543b49 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -498,6 +498,128 @@ impl<'e> NamespaceHandle<'e> { .with_resolver_mut(&self.id, |r| r.update_namespace(edit))? } + /// List all conjunction rules currently active on this namespace. + pub fn list_policy_overrides(&self) -> Vec { + self.engine + .with_resolver(&self.id, |r| r.index().policy_overrides.clone()) + } + + /// Append a new conjunction rule. Validates: ≥2 distinct words, intent must + /// exist in the namespace, bonus > 0. Returns the new rule's index. + pub fn add_policy_override( + &self, + words: Vec, + intent: String, + bonus: f32, + ) -> Result { + // Normalise + dedupe words; lowercase to match tokenizer output. + let mut normalised: Vec = words + .iter() + .map(|w| w.trim().to_lowercase()) + .filter(|w| !w.is_empty()) + .collect(); + normalised.sort(); + normalised.dedup(); + if normalised.len() < 2 { + return Err(Error::Parse( + "conjunction needs at least 2 distinct non-empty words".into(), + )); + } + if intent.trim().is_empty() { + return Err(Error::Parse("conjunction intent must not be empty".into())); + } + if bonus <= 0.0 { + return Err(Error::Parse("conjunction bonus must be > 0".into())); + } + let intent_clone = intent.clone(); + self.engine.with_resolver_mut(&self.id, |r| { + // Intent must exist (training or description). + if r.training(&intent_clone).is_none() { + return Err(Error::Parse(format!( + "conjunction targets unknown intent '{}'", + intent_clone + ))); + } + r.index_mut() + .policy_overrides + .push(crate::scoring::PolicyOverride { + words: normalised.clone(), + intent: intent_clone, + bonus, + }); + Ok(r.index().policy_overrides.len() - 1) + })? + } + + /// Remove the conjunction at the given index. Returns the removed rule. + pub fn remove_policy_override( + &self, + idx: usize, + ) -> Result { + self.engine.with_resolver_mut(&self.id, |r| { + let rules = &mut r.index_mut().policy_overrides; + if idx >= rules.len() { + return Err(Error::Parse(format!( + "conjunction index {} out of range (len={})", + idx, + rules.len() + ))); + } + Ok(rules.remove(idx)) + })? + } + + /// Replace the conjunction at the given index. Same validation as `add`. + pub fn update_policy_override( + &self, + idx: usize, + words: Vec, + intent: String, + bonus: f32, + ) -> Result<(), Error> { + let mut normalised: Vec = words + .iter() + .map(|w| w.trim().to_lowercase()) + .filter(|w| !w.is_empty()) + .collect(); + normalised.sort(); + normalised.dedup(); + if normalised.len() < 2 { + return Err(Error::Parse( + "conjunction needs at least 2 distinct non-empty words".into(), + )); + } + if intent.trim().is_empty() { + return Err(Error::Parse("conjunction intent must not be empty".into())); + } + if bonus <= 0.0 { + return Err(Error::Parse("conjunction bonus must be > 0".into())); + } + let intent_clone = intent.clone(); + self.engine.with_resolver_mut(&self.id, |r| { + if r.training(&intent_clone).is_none() { + return Err(Error::Parse(format!( + "conjunction targets unknown intent '{}'", + intent_clone + ))); + } + let rules = &mut r.index_mut().policy_overrides; + if idx >= rules.len() { + return Err(Error::Parse(format!( + "conjunction index {} out of range (len={})", + idx, + rules.len() + ))); + } + rules[idx] = crate::scoring::PolicyOverride { + words: normalised.clone(), + intent: intent_clone, + bonus, + }; + Ok(()) + })? + } + /// Export resolver state as a JSON string (for sync/backup). pub fn export_json(&self) -> String { self.engine.with_resolver(&self.id, |r| r.export_json()) @@ -756,7 +878,7 @@ impl<'e> NamespaceHandle<'e> { let (result, trace) = self.engine.with_resolver(&self.id, |r| { let threshold = r.resolve_threshold(None, crate::DEFAULT_THRESHOLD); let tokens: Vec = crate::tokenizer::tokenize(query); - let (raw, negated) = r.index().score(query); + let (raw, negated, per_token, per_intent) = r.index().score_with_attribution(query); let (multi, _neg2, multi_trace) = r.index().score_multi_with_trace( query, candidate_threshold(threshold), @@ -764,12 +886,16 @@ impl<'e> NamespaceHandle<'e> { ); let result = build_resolve_result(multi, raw.clone(), negated, tokens.clone(), threshold); + let explanation = build_explanation(&per_intent, threshold); let trace = crate::ResolveTrace { tokens, all_scores: raw, multi_round_trace: multi_trace, negated, threshold_applied: threshold, + per_token, + per_intent, + explanation, }; (result, trace) }); @@ -793,18 +919,31 @@ impl<'e> NamespaceHandle<'e> { let (raw, negated) = r.index().score(query); let scoring_threshold = candidate_threshold(threshold); if with_trace { + let (raw_attr, negated_attr, per_token, per_intent) = + r.index().score_with_attribution(query); let (multi, _neg2, multi_trace) = r.index() .score_multi_with_trace(query, scoring_threshold, gap); - let result = - build_resolve_result(multi, raw.clone(), negated, tokens.clone(), threshold); + let result = build_resolve_result( + multi, + raw_attr.clone(), + negated_attr, + tokens.clone(), + threshold, + ); + let explanation = build_explanation(&per_intent, threshold); let trace = crate::ResolveTrace { tokens, - all_scores: raw, + all_scores: raw_attr, multi_round_trace: multi_trace, - negated, + negated: negated_attr, threshold_applied: threshold, + per_token, + per_intent, + explanation, }; + let _ = raw; + let _ = negated; (result, Some(trace)) } else { let (multi, _neg2) = r.index().score_multi(query, scoring_threshold, gap); @@ -992,6 +1131,14 @@ pub struct ResolveTrace { pub multi_round_trace: crate::scoring::MultiIntentTrace, pub negated: bool, pub threshold_applied: f32, + /// Per-token contribution to each intent it activates. Sum of (delta) for + /// a given intent equals that intent's raw score before the voting gate. + pub per_token: Vec, + /// Per-intent summary capped at top 10 by score: raw score, voting state, + /// conjunction bonuses + which rules fired. + pub per_intent: Vec, + /// Single-line human-readable explanation of the routing decision. + pub explanation: String, } // ── build_resolve_result helper ───────────────────────────────────────── @@ -1050,6 +1197,50 @@ fn build_resolve_result( } } +/// Build a one-line human-readable explanation of a routing decision from +/// the per-intent trace summary. Used by [`ResolveTrace::explanation`]. +fn build_explanation(per_intent: &[crate::scoring::IntentTraceSummary], threshold: f32) -> String { + if per_intent.is_empty() { + return "No intent activated above zero. No tokens in the query matched the index." + .to_string(); + } + let top = &per_intent[0]; + let next_score = per_intent.get(1).map(|s| s.raw_score).unwrap_or(0.0); + let band = if top.raw_score >= threshold { + "High" + } else if top.raw_score >= (threshold * 0.2).max(0.05) { + "Medium" + } else { + "Low" + }; + let mut parts = vec![format!( + "{} won at {:.2} ({} band, threshold {:.2})", + top.intent, top.raw_score, band, threshold + )]; + parts.push(format!( + "{} voting token{}", + top.voting_tokens, + if top.voting_tokens == 1 { "" } else { "s" } + )); + if (top.voting_multiplier - 1.0).abs() > 1e-6 { + parts.push(format!("voting multiplier ×{:.2}", top.voting_multiplier)); + } + if !top.policy_overrides_fired.is_empty() { + parts.push(format!( + "conjunctions {}", + top.policy_overrides_fired.join(", ") + )); + } + if next_score > 0.0 { + parts.push(format!( + "beat next ({}) by {:.2}", + per_intent.get(1).map(|s| s.intent.as_str()).unwrap_or("?"), + top.raw_score - next_score + )); + } + parts.join(" · ") +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/resolver_core.rs b/src/resolver_core.rs index e28567a..fbf88fc 100644 --- a/src/resolver_core.rs +++ b/src/resolver_core.rs @@ -244,7 +244,15 @@ impl Resolver { /// namespace. Clears the existing index, re-indexes every stored /// phrase, and wipes the negative-training audit log. pub fn rebuild_index(&mut self) { + // Preserve declarative compositional rules across rebuild — they're + // pack-author authored, not derived from training phrases. Without + // this preservation, every rebuild_index drops conjunctions back + // to the empty default. + let saved_overrides = std::mem::take(&mut self.index.policy_overrides); + let saved_min_voting = self.index.min_voting_tokens; self.index = crate::scoring::IntentIndex::new(); + self.index.policy_overrides = saved_overrides; + self.index.min_voting_tokens = saved_min_voting; let all: Vec<(String, String)> = self .training .iter() diff --git a/src/resolver_persist.rs b/src/resolver_persist.rs index d94427b..8c75d18 100644 --- a/src/resolver_persist.rs +++ b/src/resolver_persist.rs @@ -121,6 +121,53 @@ impl Resolver { } } + // Policy override rules: declarative hard policy per pack — the + // narrow escape hatch for externally-specified rules that the + // auto-learn loop cannot teach quickly enough (Article 5 carve-outs, + // CSAM detection vs generation, similar). Usage guideline: ≤10 per + // pack. Mechanism is a token conjunction (all listed words must + // appear); role is policy override. + // + // Format in _ns.json: + // "policy_overrides": [{"words":[...], "intent":"...", "bonus":N}] + // + // Loaded into a local Vec here; applied below AFTER _index.json + // load so the override below doesn't drop them. + let mut ns_overrides: Vec = Vec::new(); + if let Ok(json) = std::fs::read_to_string(path.join("_ns.json")) { + if let Ok(val) = serde_json::from_str::(&json) { + if let Some(rules) = val.get("policy_overrides").and_then(|c| c.as_array()) { + for rule_val in rules { + let words: Vec = rule_val + .get("words") + .and_then(|w| w.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(|s| s.to_lowercase())) + .collect() + }) + .unwrap_or_default(); + let intent = rule_val + .get("intent") + .and_then(|i| i.as_str()) + .unwrap_or("") + .to_string(); + let bonus = rule_val + .get("bonus") + .and_then(|b| b.as_f64()) + .unwrap_or(0.0) as f32; + if words.len() >= 2 && !intent.is_empty() && bonus > 0.0 { + ns_overrides.push(crate::scoring::PolicyOverride { + words, + intent, + bonus, + }); + } + } + } + } + } + // Always rebuild the IntentIndex from seeds at load time. No // persistent index cache — the rebuild is sub-millisecond for the // pack sizes we ship and removing the cache eliminates a class of @@ -156,6 +203,13 @@ impl Resolver { router.index.set_min_voting_tokens(v); } + // Apply _ns.json conjunctions AFTER _index.json overwrite so they + // always reflect the pack author's declared rules. _ns.json is + // source of truth for compositional logic. + if !ns_overrides.is_empty() { + router.index.policy_overrides = ns_overrides; + } + let entries = std::fs::read_dir(path).map_err(|e| { crate::Error::Persistence(format!("cannot read {}: {}", path.display(), e)) })?; @@ -268,13 +322,27 @@ impl Resolver { ns_meta["lexical_groups"] = serde_json::to_value(&self.lexical_groups) .unwrap_or_else(|_| serde_json::json!([])); } else if ns_meta.get("lexical_groups").is_some() { - // Operator removed all groups → drop the field instead of writing [] - // so the file stays clean. if let Some(obj) = ns_meta.as_object_mut() { obj.remove("lexical_groups"); } } + // Persist policy override rules so authored rules survive save/load. + if !self.index.policy_overrides.is_empty() { + let rules: Vec = self + .index + .policy_overrides + .iter() + .map(|r| { + serde_json::json!({ + "words": r.words, + "intent": r.intent, + "bonus": r.bonus, + }) + }) + .collect(); + ns_meta["policy_overrides"] = serde_json::Value::Array(rules); + } std::fs::write( path.join("_ns.json"), serde_json::to_string_pretty(&ns_meta).unwrap_or_default(), diff --git a/src/scoring.rs b/src/scoring.rs index 9674906..d0c046d 100644 --- a/src/scoring.rs +++ b/src/scoring.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; /// A conjunction rule fires when ALL listed words appear in the normalized query. /// Adds a bonus activation to the target intent on top of individual word weights. #[derive(Serialize, Deserialize, Clone, Debug)] -pub struct ConjunctionRule { +pub struct PolicyOverride { pub words: Vec, pub intent: String, pub bonus: f32, @@ -47,6 +47,31 @@ pub struct MultiIntentTrace { pub stop_reason: String, } +/// Per-token contribution to a specific intent's score during scoring. +/// Emitted when full-trace mode is enabled; the sum of (delta) across tokens +/// for a given intent equals that intent's raw score before the voting gate. +#[derive(Serialize, Clone, Debug)] +pub struct TokenContribution { + pub token: String, + pub intent: String, + pub weight: f32, + pub idf: f32, + pub delta: f32, + pub negated: bool, +} + +/// Per-intent summary for full-trace output. Captures the IDF score, the +/// voting-gate state, conjunction bonuses, and any rules that fired. +#[derive(Serialize, Clone, Debug)] +pub struct IntentTraceSummary { + pub intent: String, + pub raw_score: f32, + pub voting_tokens: usize, + pub voting_multiplier: f32, + pub policy_overrides_bonus: f32, + pub policy_overrides_fired: Vec, +} + #[derive(Serialize, Deserialize, Clone, Debug, Default)] pub struct IntentIndex { /// word → [(intent_id, weight 0.0–1.0)] @@ -55,7 +80,7 @@ pub struct IntentIndex { /// Conjunction bonuses — word pairs that together strongly indicate an intent. #[serde(default)] - pub conjunctions: Vec, + pub policy_overrides: Vec, /// Char-ngram tiebreaker index: intent_id → set of char 4-grams from seed phrases. #[serde(default)] @@ -360,7 +385,7 @@ impl IntentIndex { pub fn fired_conjunction_indices(&self, words: &[&str]) -> Vec { let word_set: FxHashSet<&str> = words.iter().copied().collect(); - self.conjunctions + self.policy_overrides .iter() .enumerate() .filter(|(_, rule)| rule.words.iter().all(|w| word_set.contains(w.as_str()))) @@ -369,7 +394,7 @@ impl IntentIndex { } pub fn reinforce_conjunction(&mut self, idx: usize, delta: f32) { - if let Some(rule) = self.conjunctions.get_mut(idx) { + if let Some(rule) = self.policy_overrides.get_mut(idx) { if delta >= 0.0 { rule.bonus = (rule.bonus + delta * (1.0 - rule.bonus)).min(1.0); } else { @@ -431,7 +456,7 @@ impl IntentIndex { } } - for rule in &self.conjunctions { + for rule in &self.policy_overrides { if rule.words.iter().all(|w| all_bases.contains(w.as_str())) { *scores.entry(rule.intent.clone()).or_insert(0.0) += rule.bonus; } @@ -462,6 +487,140 @@ impl IntentIndex { (result, has_negation) } + /// Like [`score`] but also emits per-token contribution data and per-intent + /// summaries. Used by full-trace mode in the resolver pipeline. Slower than + /// `score` because it allocates the contribution vector; the no-trace path + /// stays unchanged. + #[allow(clippy::type_complexity)] + pub fn score_with_attribution( + &self, + normalized: &str, + ) -> ( + Vec<(String, f32)>, + bool, + Vec, + Vec, + ) { + const CJK_NEG: &[char] = &['不', '没', '别', '未']; + let cjk_negated = normalized.chars().any(|c| CJK_NEG.contains(&c)); + let query_for_tokenize: std::borrow::Cow = if cjk_negated { + std::borrow::Cow::Owned( + normalized + .chars() + .map(|c| if CJK_NEG.contains(&c) { ' ' } else { c }) + .collect(), + ) + } else { + std::borrow::Cow::Borrowed(normalized) + }; + + let tokens = crate::tokenizer::tokenize(&query_for_tokenize); + let mut scores: FxHashMap = FxHashMap::default(); + let mut has_negation = cjk_negated; + let mut voting_pairs: FxHashSet<(String, String)> = FxHashSet::default(); + let mut contributions: Vec = Vec::new(); + let mut policy_overrides_bonus: FxHashMap = FxHashMap::default(); + let mut policy_overrides_fired: FxHashMap> = FxHashMap::default(); + const VOTING_EPSILON: f32 = 0.05; + + let all_bases: FxHashSet<&str> = tokens + .iter() + .map(|t| t.strip_prefix("not_").unwrap_or(t.as_str())) + .collect(); + + for token in &tokens { + let is_negated = token.starts_with("not_"); + let base = if is_negated { + &token["not_".len()..] + } else { + token.as_str() + }; + if is_negated { + has_negation = true; + } + if let Some(activations) = self.word_intent.get(base) { + let idf = self.idf(base); + for (intent, weight) in activations { + let delta = weight * idf; + let signed = if is_negated { -delta } else { delta }; + *scores.entry(intent.clone()).or_insert(0.0) += signed; + if !is_negated && delta > VOTING_EPSILON { + voting_pairs.insert((intent.clone(), base.to_string())); + } + contributions.push(TokenContribution { + token: base.to_string(), + intent: intent.clone(), + weight: *weight, + idf, + delta: signed, + negated: is_negated, + }); + } + } + } + + for rule in &self.policy_overrides { + if rule.words.iter().all(|w| all_bases.contains(w.as_str())) { + *scores.entry(rule.intent.clone()).or_insert(0.0) += rule.bonus; + *policy_overrides_bonus + .entry(rule.intent.clone()) + .or_insert(0.0) += rule.bonus; + policy_overrides_fired + .entry(rule.intent.clone()) + .or_default() + .push(format!("[{}]", rule.words.join(" + "))); + } + } + + // Voting-token state per intent (pre-gate); the gate itself rescales + // scores below. + let mut voting_count: FxHashMap = FxHashMap::default(); + for (intent, _) in &voting_pairs { + *voting_count.entry(intent.clone()).or_insert(0) += 1; + } + let mut voting_mult: FxHashMap = FxHashMap::default(); + + if self.min_voting_tokens > 1 { + let min = self.min_voting_tokens as usize; + let mut updates: Vec<(String, f32)> = Vec::new(); + for (intent, score) in scores.iter() { + let count = voting_count.get(intent).copied().unwrap_or(0); + let multiplier = voting_multiplier(count, min); + voting_mult.insert(intent.clone(), multiplier); + if (multiplier - 1.0).abs() > 1e-6 { + updates.push((intent.clone(), score * multiplier)); + } + } + for (intent, new_score) in updates { + scores.insert(intent, new_score); + } + } else { + for intent in scores.keys() { + voting_mult.insert(intent.clone(), 1.0); + } + } + + let mut result: Vec<(String, f32)> = scores.into_iter().filter(|(_, s)| *s > 0.0).collect(); + result.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + // Build per-intent summary for the top entries (cap at 10 to keep + // payloads small; UI/audit can expand if needed). + let summary: Vec = result + .iter() + .take(10) + .map(|(id, score)| IntentTraceSummary { + intent: id.clone(), + raw_score: *score, + voting_tokens: voting_count.get(id).copied().unwrap_or(0), + voting_multiplier: voting_mult.get(id).copied().unwrap_or(1.0), + policy_overrides_bonus: policy_overrides_bonus.get(id).copied().unwrap_or(0.0), + policy_overrides_fired: policy_overrides_fired.get(id).cloned().unwrap_or_default(), + }) + .collect(); + + (result, has_negation, contributions, summary) + } + pub fn score_multi( &self, normalized: &str, @@ -634,7 +793,7 @@ impl IntentIndex { .iter() .map(|t| t.strip_prefix("not_").unwrap_or(t.as_str())) .collect(); - for rule in &self.conjunctions { + for rule in &self.policy_overrides { if !exclude_intents.contains(&rule.intent) && rule.words.iter().all(|w| all_bases.contains(w.as_str())) { @@ -794,7 +953,7 @@ impl IntentIndex { ( self.word_intent.len(), activation_edges, - self.conjunctions.len(), + self.policy_overrides.len(), ) } } diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 6f77a21..9fefb26 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -7,6 +7,7 @@ import SimulatePage from '@/pages/SimulatePage'; import ReviewPage from '@/pages/ReviewPage'; import IntentsPage from '@/pages/IntentsPage'; import LexicalGroupsPage from '@/pages/LexicalGroupsPage'; +import PolicyOverridesPage from '@/pages/PolicyOverridesPage'; import SettingsPage from '@/pages/SettingsPage'; import NamespacesPage from '@/pages/NamespacesPage'; import ModelsPage from '@/pages/ModelsPage'; @@ -125,6 +126,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 01c36f4..d421d39 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -170,9 +170,37 @@ export interface ResolveIntent { band: BandLabel; } +export interface TokenContribution { + token: string; + intent: string; + weight: number; + idf: number; + delta: number; + negated: boolean; +} + +export interface IntentTraceSummary { + intent: string; + raw_score: number; + voting_tokens: number; + voting_multiplier: number; + policy_overrides_bonus: number; + policy_overrides_fired: string[]; +} + export interface ResolveTrace { tokens: string[]; - [key: string]: unknown; + all_scores: { id: string; score: number }[]; + per_token: TokenContribution[]; + per_intent: IntentTraceSummary[]; + explanation: string; + threshold_applied: number; + negated: boolean; + multi?: { + rounds: unknown[]; + stop_reason: string; + has_negation: boolean; + }; } export interface ResolveOutput { @@ -182,6 +210,13 @@ export interface ResolveOutput { trace?: ResolveTrace; } +export interface PolicyOverrideRow { + idx: number; + words: string[]; + intent: string; + bonus: number; +} + export interface NamespaceModel { label: string; model_id: string; @@ -261,8 +296,22 @@ export const api = { health: () => get('/health'), // Routing - resolve: (query: string, threshold = 0.3, log = true) => - post('/resolve', { query, threshold, log }), + resolve: (query: string, threshold = 0.3, log = true, trace = false) => + post('/resolve', { query, threshold, log, trace }), + + // Policy overrides — narrow declarative escape hatch (≤10 per pack). + // Hard rules pack authors encode for externally-specified policy that + // the auto-learn loop cannot reasonably teach (Article 5 carve-outs, + // CSAM detection vs generation, similar). Mechanism is a token + // conjunction; role is policy override. + listPolicyOverrides: () => + get<{ policy_overrides: PolicyOverrideRow[] }>('/policy-overrides'), + addPolicyOverride: (payload: { words: string[]; intent: string; bonus: number }) => + post<{ idx: number }>('/policy-overrides', payload), + removePolicyOverride: (idx: number) => + del(`/policy-overrides/${idx}`), + updatePolicyOverride: (idx: number, payload: { words: string[]; intent: string; bonus: number }) => + patch(`/policy-overrides/${idx}`, payload), // Intents listIntents: () => get('/intents'), diff --git a/ui/src/components/Layout.tsx b/ui/src/components/Layout.tsx index 05c970e..a66c76f 100644 --- a/ui/src/components/Layout.tsx +++ b/ui/src/components/Layout.tsx @@ -182,6 +182,8 @@ export default function Layout() { hint: 'Manage intents, training phrases, metadata' }, { to: '/lexical', label: 'Lexicon', icon: '⌥', hint: 'Per-namespace morph + abbrev normalization' }, + { to: '/policy-overrides', label: 'Policy overrides', icon: '⚖', + hint: 'Hard rules — externally-specified policy assertions, ≤10 per pack' }, ], }, { diff --git a/ui/src/pages/PolicyOverridesPage.tsx b/ui/src/pages/PolicyOverridesPage.tsx new file mode 100644 index 0000000..53e24d8 --- /dev/null +++ b/ui/src/pages/PolicyOverridesPage.tsx @@ -0,0 +1,267 @@ +import { useEffect, useState } from 'react'; +import { api, type PolicyOverrideRow, type IntentInfo, type ResolveOutput } from '@/api/client'; +import Page from '@/components/Page'; + +/// Policy overrides editor — narrow declarative escape hatch (≤10 per pack). +/// +/// A policy override fires when ALL listed words appear in the normalised +/// query, adding `bonus` to the target intent. Designed for hard rules pack +/// authors encode for externally-specified policy that the auto-learn loop +/// cannot reasonably teach (Article 5 carve-outs, CSAM detection vs +/// generation, similar). Mechanism is a token conjunction; role is policy +/// override. Every mutation lands in the audit log. +export default function PolicyOverridesPage() { + const [rows, setRows] = useState([]); + const [intents, setIntents] = useState([]); + const [loading, setLoading] = useState(true); + const [err, setErr] = useState(null); + + // New rule form + const [newWords, setNewWords] = useState(''); + const [newIntent, setNewIntent] = useState(''); + const [newBonus, setNewBonus] = useState(2.0); + + // Live preview + const [previewQuery, setPreviewQuery] = useState(''); + const [previewResult, setPreviewResult] = useState(null); + const [previewLoading, setPreviewLoading] = useState(false); + + const refresh = async () => { + setLoading(true); + try { + const [r, ils] = await Promise.all([api.listPolicyOverrides(), api.listIntents()]); + setRows(r.policy_overrides || []); + setIntents(ils); + if (!newIntent && ils.length > 0) setNewIntent(ils[0].id); + setErr(null); + } catch (e) { + setErr(String(e)); + } finally { + setLoading(false); + } + }; + + useEffect(() => { refresh(); }, []); // eslint-disable-line react-hooks/exhaustive-deps + + const addRule = async () => { + const words = newWords + .split(/[\s,]+/) + .map(w => w.trim().toLowerCase()) + .filter(w => w.length > 0); + if (words.length < 2) { + setErr('Need at least 2 distinct words'); + return; + } + if (!newIntent) { + setErr('Pick an intent'); + return; + } + try { + await api.addPolicyOverride({ words, intent: newIntent, bonus: newBonus }); + setNewWords(''); + setNewBonus(2.0); + setErr(null); + refresh(); + if (previewQuery) runPreview(previewQuery); + } catch (e) { + setErr(String(e)); + } + }; + + const deleteRule = async (idx: number) => { + if (!confirm('Remove this policy override?')) return; + try { + await api.removePolicyOverride(idx); + refresh(); + if (previewQuery) runPreview(previewQuery); + } catch (e) { + setErr(String(e)); + } + }; + + const runPreview = async (q: string) => { + setPreviewLoading(true); + try { + const r = await api.resolve(q, 0.3, false, true); + setPreviewResult(r); + } catch (e) { + setErr(String(e)); + } finally { + setPreviewLoading(false); + } + }; + + // Which policy overrides fire on the current preview? + // The trace's per_intent[].policy_overrides_fired contains rule descriptions + // (the runtime mechanism is still a conjunction) — match against rules. + const firedRuleSigs = new Set(); + if (previewResult?.trace?.per_intent) { + for (const pi of previewResult.trace.per_intent) { + for (const f of pi.policy_overrides_fired) { + // f is like "[word_a + word_b]" — extract sorted words. + const inner = f.replace(/[\[\]]/g, ''); + const ws = inner.split(' + ').map(w => w.trim()).sort(); + firedRuleSigs.add(`${pi.intent}::${ws.join(' + ')}`); + } + } + } + const ruleFired = (r: PolicyOverrideRow): boolean => { + const sig = `${r.intent}::${[...r.words].sort().join(' + ')}`; + return firedRuleSigs.has(sig); + }; + + return ( + +
+ {err && ( +
+ {err} +
+ )} + + {/* What is this */} +
+ A policy override fires when all listed words appear in a query, adding the bonus to the target intent. Reserved for hard rules where pre-knowledge of policy is more efficient than waiting for the auto-learn loop to discover it (Article 5 carve-outs, CSAM detection vs generation, similar). Use sparingly — keep ≤10 per pack. Every add / remove is recorded in the audit log. +
+ + {/* New rule form */} +
+
Add rule
+
+
+ + setNewWords(e.target.value)} + placeholder="missing, child" + className="w-full bg-zinc-950 border border-zinc-700 rounded px-2 py-1.5 text-zinc-100 font-mono text-xs" + /> +
+
+ + +
+
+ + setNewBonus(parseFloat(e.target.value) || 0)} + className="w-20 bg-zinc-950 border border-zinc-700 rounded px-2 py-1.5 text-zinc-100 font-mono text-xs" + /> +
+ +
+
+ + {/* Preview tool */} +
+
+ Test query — see which overrides fire +
+
+ setPreviewQuery(e.target.value)} + onKeyDown={e => { if (e.key === 'Enter') runPreview(previewQuery); }} + placeholder="e.g. live face match for missing child" + className="flex-1 bg-zinc-950 border border-zinc-700 rounded px-2 py-1.5 text-zinc-100 font-mono text-xs" + /> + +
+ {previewResult && ( +
+
+ Top intent:{' '} + + {previewResult.intents[0]?.id ?? '(no match)'} + + {previewResult.intents[0] && ( + + {previewResult.intents[0].score.toFixed(2)} + + )} +
+
+ Fired overrides are highlighted in the rule list below. +
+
+ )} +
+ + {/* Rule list */} +
+
+ Active rules ({rows.length}) +
+ {loading && ( +
Loading…
+ )} + {!loading && rows.length === 0 && ( +
+ No policy overrides yet. Add one above (sparingly). +
+ )} + {rows.map(r => ( +
+ #{r.idx} + + {r.words.map((w, i) => ( + + {i > 0 && + } + {w} + + ))} + + → + {r.intent} + +{r.bonus.toFixed(2)} + {ruleFired(r) && ( + + fired + + )} + +
+ ))} +
+
+ + ); +} diff --git a/ui/src/pages/RouterPage.tsx b/ui/src/pages/RouterPage.tsx index e608cc7..62be99a 100644 --- a/ui/src/pages/RouterPage.tsx +++ b/ui/src/pages/RouterPage.tsx @@ -1,5 +1,5 @@ import { useState, useRef, useEffect } from 'react'; -import { api, type ResolveOutput, type ReviewAnalysis } from '@/api/client'; +import { api, type ResolveOutput, type ReviewAnalysis, type ResolveTrace } from '@/api/client'; import Page from '@/components/Page'; const INTENT_COLORS = [ @@ -45,11 +45,11 @@ export default function RouterPage() { }; const handleInput = async (raw: string) => { - // Regular query + // Regular query — request trace so the "why?" panel has data to show. push({ type: 'query', text: raw }); const t0 = performance.now(); try { - const result = await api.resolve(raw, 0.3, true); + const result = await api.resolve(raw, 0.3, true, true); const latency = performance.now() - t0; push({ type: 'result', result, latency, query: raw }); } catch (err) { @@ -256,6 +256,9 @@ function MessageBubble({ msg, onApplySuggestion, onTrain, intentCount }: { + {/* Why? — per-token and per-intent attribution. */} + {result.trace && } + {/* E1: LLM Review card */} {reviewing && (
@@ -270,6 +273,112 @@ function MessageBubble({ msg, onApplySuggestion, onTrain, intentCount }: { ); } +// --- Trace panel --- + +function TracePanel({ trace }: { trace: ResolveTrace }) { + const [open, setOpen] = useState(false); + + const intentColor = (i: number) => INTENT_COLORS[i % INTENT_COLORS.length]; + + // Group per-token contributions: token → list of (intent, delta) sorted desc + const groups: { token: string; entries: { intent: string; delta: number; idf: number; weight: number; negated: boolean }[] }[] = []; + const seen = new Set(); + for (const c of trace.per_token) { + if (!seen.has(c.token)) { + seen.add(c.token); + groups.push({ token: c.token, entries: [] }); + } + const g = groups.find(x => x.token === c.token)!; + g.entries.push({ intent: c.intent, delta: c.delta, idf: c.idf, weight: c.weight, negated: c.negated }); + } + for (const g of groups) g.entries.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta)); + + return ( +
+ + + {open && ( +
+ {/* Per-intent summary */} +
+
Per-intent score breakdown
+ + + + + + + + + + + + {trace.per_intent.map((s, i) => ( + + + + + + + + ))} + +
IntentScoreVoting×MultConjunctions
{s.intent}{s.raw_score.toFixed(2)}{s.voting_tokens} 0.01 ? 'text-amber-400' : 'text-zinc-600'}`}> + {s.voting_multiplier.toFixed(2)} + + {s.policy_overrides_fired.length > 0 ? s.policy_overrides_fired.join(', ') : —} +
+
+ + {/* Per-token attribution */} +
+
Per-token contribution (delta = weight × IDF, signed)
+
+ {groups.map(g => ( +
+ + {g.entries[0].negated ? `¬${g.token}` : g.token} + +
+ {g.entries.map((e, idx) => ( + + p.intent === e.intent) >= 0 + ? intentColor(trace.per_intent.findIndex(p => p.intent === e.intent)) + : 'text-zinc-600'}> + {e.intent} + + + {' '}{e.delta >= 0 ? '+' : ''}{e.delta.toFixed(2)} + + + ))} +
+
+ ))} + {groups.length === 0 && ( +
No tokens activated any intent.
+ )} +
+
+ + {/* Threshold context */} +
+ Threshold applied: {trace.threshold_applied.toFixed(2)} + {trace.negated && negation detected} +
+
+ )} +
+ ); +} + // --- Intent row --- const BAND_STYLES: Record = {