From 2b26eb3f474f9caec79160ea108473b34ed39395 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Mar 2026 10:57:05 +0000 Subject: [PATCH 1/2] Add fault injection feature for chaos/failure testing Introduces a --fault CLI flag (proxy backend only) that injects artificial delays or synthetic HTTP error responses into proxied requests, enabling chaos-style failure testing without modifying the target application. Supported rule formats: delay:100ms fixed latency delay:100ms-500ms random latency in range delay:200ms:/api latency only for matching URLs error:503 always return HTTP 503 error:503:0.5 50% probability of HTTP 503 error:500:0.1:/api 10% error on URLs containing "/api" Multiple rules can be combined with repeated --fault flags. Fault-injected traces are recorded in the store with the synthetic status code for later analysis. https://claude.ai/code/session_013JsZoVqikbBRzwiLjDozKM --- crates/phantom-capture/src/fault.rs | 242 ++++++++++++++++++++++++++++ crates/phantom-capture/src/lib.rs | 2 + crates/phantom-capture/src/proxy.rs | 81 +++++++++- src/main.rs | 36 ++++- 4 files changed, 358 insertions(+), 3 deletions(-) create mode 100644 crates/phantom-capture/src/fault.rs diff --git a/crates/phantom-capture/src/fault.rs b/crates/phantom-capture/src/fault.rs new file mode 100644 index 0000000..61d0e0f --- /dev/null +++ b/crates/phantom-capture/src/fault.rs @@ -0,0 +1,242 @@ +/// A single fault injection rule, evaluated per request. +#[derive(Clone, Debug)] +pub enum FaultRule { + /// Inject artificial latency before forwarding the request. + Delay { + min_ms: u64, + max_ms: u64, + /// If Some, only applies when the URL contains this substring. + url_pattern: Option, + }, + /// Return a synthetic HTTP error response instead of forwarding. + Error { + status_code: u16, + /// Probability 0.0–1.0 (1.0 = always inject). + probability: f64, + /// If Some, only applies when the URL contains this substring. + url_pattern: Option, + }, +} + +impl FaultRule { + /// Returns true if this rule applies to the given URL. + pub fn matches_url(&self, url: &str) -> bool { + let pattern = match self { + FaultRule::Delay { url_pattern, .. } => url_pattern, + FaultRule::Error { url_pattern, .. } => url_pattern, + }; + pattern.as_ref().map(|p| url.contains(p.as_str())).unwrap_or(true) + } +} + +/// A collection of fault rules applied in order to each proxied request. +#[derive(Clone, Debug, Default)] +pub struct FaultConfig { + pub rules: Vec, +} + +// ───────────────────────────────────────────────────────────────────────────── +// CLI spec parsing +// ───────────────────────────────────────────────────────────────────────────── + +/// Parse a fault specification string into a `FaultRule`. +/// +/// Formats: +/// delay:100ms fixed 100ms delay on all requests +/// delay:100ms-500ms random delay in 100–500ms range +/// delay:100ms:/api delay only URLs containing "/api" +/// delay:100ms-500ms:/api range delay with URL filter +/// error:503 always return HTTP 503 +/// error:503:0.5 return HTTP 503 with 50% probability +/// error:503:/api always return 503 for URLs containing "/api" +/// error:503:0.5:/api probability + URL filter +pub fn parse_fault_spec(s: &str) -> Result { + let (kind, rest) = s + .split_once(':') + .ok_or_else(|| format!("invalid fault spec {s:?}: expected 'delay:…' or 'error:…'"))?; + match kind { + "delay" => parse_delay(rest), + "error" => parse_error(rest), + _ => Err(format!( + "unknown fault type {kind:?} in {s:?}; expected 'delay' or 'error'" + )), + } +} + +fn parse_delay(rest: &str) -> Result { + let (timing, url_pattern) = split_url_suffix(rest); + if let Some(dash) = timing.find('-') { + let min_ms = parse_ms(&timing[..dash])?; + let max_ms = parse_ms(&timing[dash + 1..])?; + if min_ms > max_ms { + return Err(format!( + "delay range min ({min_ms}ms) must not exceed max ({max_ms}ms)" + )); + } + Ok(FaultRule::Delay { + min_ms, + max_ms, + url_pattern, + }) + } else { + let ms = parse_ms(timing)?; + Ok(FaultRule::Delay { + min_ms: ms, + max_ms: ms, + url_pattern, + }) + } +} + +fn parse_error(rest: &str) -> Result { + // Optional URL pattern is the last colon-segment starting with '/'. + let (url_pattern, non_url) = if let Some(pos) = rest.rfind(':') { + if rest[pos + 1..].starts_with('/') { + (Some(rest[pos + 1..].to_string()), &rest[..pos]) + } else { + (None, rest) + } + } else { + (None, rest) + }; + + let parts: Vec<&str> = non_url.splitn(2, ':').collect(); + let status_code: u16 = parts[0] + .parse() + .map_err(|_| format!("invalid HTTP status code {:?}", parts[0]))?; + if !(100..=599).contains(&status_code) { + return Err(format!( + "status code {status_code} is out of range 100–599" + )); + } + let probability: f64 = if parts.len() == 2 { + parts[1] + .parse() + .map_err(|_| format!("invalid probability {:?}; expected a float like 0.5", parts[1]))? + } else { + 1.0 + }; + if !(0.0..=1.0).contains(&probability) { + return Err(format!( + "probability {probability} is out of range 0.0–1.0" + )); + } + Ok(FaultRule::Error { + status_code, + probability, + url_pattern, + }) +} + +/// Split a trailing URL pattern (`:/`) from the rest of a spec segment. +fn split_url_suffix(s: &str) -> (&str, Option) { + if let Some(pos) = s.rfind(':') { + if s[pos + 1..].starts_with('/') { + return (&s[..pos], Some(s[pos + 1..].to_string())); + } + } + (s, None) +} + +fn parse_ms(s: &str) -> Result { + let s = s.trim(); + if let Some(n) = s.strip_suffix("ms") { + n.parse::() + .map_err(|_| format!("invalid duration {s:?}; expected e.g. '100ms'")) + } else if let Some(n) = s.strip_suffix('s') { + n.parse::() + .map(|v| v * 1000) + .map_err(|_| format!("invalid duration {s:?}; expected e.g. '2s'")) + } else { + s.parse::() + .map_err(|_| format!("invalid duration {s:?}; expected e.g. '100ms' or '2s'")) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Tests +// ───────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_fixed_delay() { + let r = parse_fault_spec("delay:200ms").unwrap(); + assert!(matches!(r, FaultRule::Delay { min_ms: 200, max_ms: 200, url_pattern: None })); + } + + #[test] + fn parse_range_delay() { + let r = parse_fault_spec("delay:100ms-500ms").unwrap(); + assert!(matches!(r, FaultRule::Delay { min_ms: 100, max_ms: 500, url_pattern: None })); + } + + #[test] + fn parse_delay_with_url() { + let r = parse_fault_spec("delay:200ms:/api/users").unwrap(); + match r { + FaultRule::Delay { min_ms: 200, max_ms: 200, url_pattern: Some(p) } => { + assert_eq!(p, "/api/users"); + } + _ => panic!("unexpected rule"), + } + } + + #[test] + fn parse_error_always() { + let r = parse_fault_spec("error:503").unwrap(); + assert!(matches!(r, FaultRule::Error { status_code: 503, .. })); + } + + #[test] + fn parse_error_probability() { + let r = parse_fault_spec("error:500:0.1").unwrap(); + match r { + FaultRule::Error { status_code: 500, probability, url_pattern: None } => { + assert!((probability - 0.1).abs() < 1e-9); + } + _ => panic!("unexpected rule"), + } + } + + #[test] + fn parse_error_probability_and_url() { + let r = parse_fault_spec("error:500:0.5:/api").unwrap(); + match r { + FaultRule::Error { status_code: 500, probability, url_pattern: Some(p) } => { + assert!((probability - 0.5).abs() < 1e-9); + assert_eq!(p, "/api"); + } + _ => panic!("unexpected rule"), + } + } + + #[test] + fn parse_seconds_duration() { + let r = parse_fault_spec("delay:2s").unwrap(); + assert!(matches!(r, FaultRule::Delay { min_ms: 2000, .. })); + } + + #[test] + fn url_pattern_matching() { + let rule = FaultRule::Delay { + min_ms: 100, + max_ms: 100, + url_pattern: Some("/api".to_string()), + }; + assert!(rule.matches_url("http://example.com/api/users")); + assert!(!rule.matches_url("http://example.com/health")); + } + + #[test] + fn no_url_pattern_matches_all() { + let rule = FaultRule::Error { + status_code: 503, + probability: 1.0, + url_pattern: None, + }; + assert!(rule.matches_url("http://example.com/anything")); + } +} diff --git a/crates/phantom-capture/src/lib.rs b/crates/phantom-capture/src/lib.rs index 39369f8..2158019 100644 --- a/crates/phantom-capture/src/lib.rs +++ b/crates/phantom-capture/src/lib.rs @@ -1,8 +1,10 @@ +pub mod fault; mod proxy; #[cfg(target_os = "linux")] mod ldpreload; +pub use fault::{parse_fault_spec, FaultConfig, FaultRule}; pub use proxy::ProxyCaptureBackend; #[cfg(target_os = "linux")] diff --git a/crates/phantom-capture/src/proxy.rs b/crates/phantom-capture/src/proxy.rs index 04fe55b..211acf3 100644 --- a/crates/phantom-capture/src/proxy.rs +++ b/crates/phantom-capture/src/proxy.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::net::SocketAddr; -use std::time::{Instant, SystemTime}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; use http::uri::Scheme; use hudsucker::certificate_authority::RcgenAuthority; @@ -13,12 +14,15 @@ use phantom_core::trace::{HttpMethod, HttpTrace, SpanId, TraceId}; use tokio::sync::{mpsc, oneshot}; use tracing::{info, warn}; +use crate::fault::{FaultConfig, FaultRule}; + /// Maximum body size to capture (1 MB). const MAX_BODY_SIZE: usize = 1024 * 1024; pub struct ProxyCaptureBackend { listen_port: u16, insecure: bool, + fault_config: FaultConfig, shutdown_tx: Option>, task_handle: Option>, } @@ -28,10 +32,17 @@ impl ProxyCaptureBackend { Self { listen_port, insecure, + fault_config: FaultConfig::default(), shutdown_tx: None, task_handle: None, } } + + /// Attach fault injection rules (builder pattern). + pub fn with_faults(mut self, config: FaultConfig) -> Self { + self.fault_config = config; + self + } } impl CaptureBackend for ProxyCaptureBackend { @@ -42,6 +53,7 @@ impl CaptureBackend for ProxyCaptureBackend { let handler = TraceHandler { trace_tx, pending: None, + fault_config: Arc::new(self.fault_config.clone()), }; let port = self.listen_port; @@ -132,6 +144,7 @@ struct TraceHandler { trace_tx: mpsc::Sender, /// Pending request info, set in handle_request, consumed in handle_response. pending: Option, + fault_config: Arc, } #[derive(Clone)] @@ -172,6 +185,72 @@ impl HttpHandler for TraceHandler { }); let rebuilt = Request::from_parts(parts, body_to_body(body_bytes)); + + // Apply fault injection rules in order. + let url = self + .pending + .as_ref() + .map(|p| p.url.clone()) + .unwrap_or_default(); + for rule in &self.fault_config.rules { + if !rule.matches_url(&url) { + continue; + } + match rule { + FaultRule::Delay { + min_ms, max_ms, .. + } => { + let delay_ms = if min_ms == max_ms { + *min_ms + } else { + min_ms + rand::random::() % (max_ms - min_ms + 1) + }; + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + FaultRule::Error { + status_code, + probability, + .. + } => { + if rand::random::() < *probability { + // Emit a trace immediately — handle_response won't be called. + if let Some(info) = self.pending.take() { + let fault_body = b"{\"fault\":\"injected\"}".to_vec(); + let trace = HttpTrace { + span_id: info.span_id, + trace_id: info.trace_id, + parent_span_id: None, + method: info.method, + url: info.url, + request_headers: info.request_headers, + request_body: info.request_body, + status_code: *status_code, + response_headers: HashMap::new(), + response_body: Some(fault_body), + timestamp: info.timestamp, + duration: info.started_at.elapsed(), + source_addr: info.source_addr, + dest_addr: None, + protocol_version: info.protocol_version, + }; + if self.trace_tx.try_send(trace).is_err() { + warn!("Trace channel full, dropping fault-injected trace"); + } + } + let body_bytes: bytes::Bytes = + b"{\"fault\":\"injected\"}".as_ref().into(); + let response = Response::builder() + .status(*status_code) + .header("content-type", "application/json") + .header("x-fault-injected", "phantom") + .body(Body::from(http_body_util::Full::new(body_bytes))) + .expect("valid fault response"); + return RequestOrResponse::Response(response); + } + } + } + } + RequestOrResponse::Request(rebuilt) } diff --git a/src/main.rs b/src/main.rs index 7ec1c75..0428a91 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::UNIX_EPOCH; use clap::{Parser, ValueEnum}; -use phantom_capture::ProxyCaptureBackend; +use phantom_capture::{FaultConfig, ProxyCaptureBackend}; use phantom_core::capture::CaptureBackend; use phantom_core::storage::TraceStore; use phantom_core::trace::HttpTrace; @@ -173,6 +173,22 @@ struct Cli { #[arg(long, value_name = "PATH")] agent_lib: Option, + /// Inject faults into proxied requests (proxy backend only). + /// + /// SPEC formats: + /// delay:100ms fixed 100 ms delay on all requests + /// delay:100ms-500ms random delay in the given range + /// delay:200ms:/api delay only URLs containing "/api" + /// error:503 return HTTP 503 for all requests + /// error:503:0.5 return HTTP 503 with 50% probability + /// error:500:0.1:/api 10% chance of HTTP 500 on URLs containing "/api" + /// + /// Rules are applied in order; delays and errors can be combined. + /// Repeat the flag to add multiple rules: + /// --fault delay:50ms --fault error:500:0.1 + #[arg(long, value_name = "SPEC")] + fault: Vec, + /// Command to spawn and trace (everything after `--`). /// /// proxy mode: HTTP_PROXY is set automatically; Node.js additionally @@ -350,6 +366,20 @@ async fn main() -> anyhow::Result<()> { } } +// ───────────────────────────────────────────────────────────────────────────── +// Fault injection +// ───────────────────────────────────────────────────────────────────────────── + +fn build_fault_config(specs: &[String]) -> anyhow::Result { + let mut rules = Vec::new(); + for spec in specs { + let rule = phantom_capture::parse_fault_spec(spec) + .map_err(|e| anyhow::anyhow!("--fault {spec:?}: {e}"))?; + rules.push(rule); + } + Ok(FaultConfig { rules }) +} + // ───────────────────────────────────────────────────────────────────────────── // Proxy backend // ───────────────────────────────────────────────────────────────────────────── @@ -418,7 +448,9 @@ fn spawn_proxy_child( } async fn run_proxy(cli: Cli, store: Arc) -> anyhow::Result<()> { - let mut backend = ProxyCaptureBackend::new(cli.port, cli.insecure); + let fault_config = build_fault_config(&cli.fault)?; + let mut backend = ProxyCaptureBackend::new(cli.port, cli.insecure) + .with_faults(fault_config); let backend_name = backend.name().to_string(); let trace_rx = backend.start().map_err(|e| anyhow::anyhow!("{e}"))?; From 19ac8966eb964e1165cc3ac55d6df64593a30a5a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Mar 2026 12:42:50 +0000 Subject: [PATCH 2/2] Add fault injection integration tests 3 new integration tests in tests/fault_injection.rs that verify --fault rules work end-to-end via the phantom CLI: - test_fault_error_always: --fault error:503 returns synthetic HTTP 503 with x-fault-injected header and {"fault":"injected"} body in trace - test_fault_delay_adds_latency: --fault delay:300ms adds >= 300ms to the trace duration_ms while forwarding the real backend response - test_fault_url_pattern_filter: --fault error:503:/api/health injects 503 only for matching URLs; unmatched URLs receive the real response Also: - Fix fault-injected trace to include synthetic response headers (content-type, x-fault-injected) so the trace record is accurate - Fix clippy/fmt issues in fault.rs Tests require curl on PATH and clear no_proxy/NO_PROXY so curl routes 127.0.0.1 requests through the phantom proxy regardless of system noproxy settings. https://claude.ai/code/session_013JsZoVqikbBRzwiLjDozKM --- crates/phantom-capture/src/fault.rs | 74 +++-- crates/phantom-capture/src/lib.rs | 2 +- crates/phantom-capture/src/proxy.rs | 17 +- src/main.rs | 3 +- tests/fault_injection.rs | 438 ++++++++++++++++++++++++++++ 5 files changed, 505 insertions(+), 29 deletions(-) create mode 100644 tests/fault_injection.rs diff --git a/crates/phantom-capture/src/fault.rs b/crates/phantom-capture/src/fault.rs index 61d0e0f..1ae25dc 100644 --- a/crates/phantom-capture/src/fault.rs +++ b/crates/phantom-capture/src/fault.rs @@ -25,7 +25,10 @@ impl FaultRule { FaultRule::Delay { url_pattern, .. } => url_pattern, FaultRule::Error { url_pattern, .. } => url_pattern, }; - pattern.as_ref().map(|p| url.contains(p.as_str())).unwrap_or(true) + pattern + .as_ref() + .map(|p| url.contains(p.as_str())) + .unwrap_or(true) } } @@ -105,21 +108,20 @@ fn parse_error(rest: &str) -> Result { .parse() .map_err(|_| format!("invalid HTTP status code {:?}", parts[0]))?; if !(100..=599).contains(&status_code) { - return Err(format!( - "status code {status_code} is out of range 100–599" - )); + return Err(format!("status code {status_code} is out of range 100–599")); } let probability: f64 = if parts.len() == 2 { - parts[1] - .parse() - .map_err(|_| format!("invalid probability {:?}; expected a float like 0.5", parts[1]))? + parts[1].parse().map_err(|_| { + format!( + "invalid probability {:?}; expected a float like 0.5", + parts[1] + ) + })? } else { 1.0 }; if !(0.0..=1.0).contains(&probability) { - return Err(format!( - "probability {probability} is out of range 0.0–1.0" - )); + return Err(format!("probability {probability} is out of range 0.0–1.0")); } Ok(FaultRule::Error { status_code, @@ -130,10 +132,10 @@ fn parse_error(rest: &str) -> Result { /// Split a trailing URL pattern (`:/`) from the rest of a spec segment. fn split_url_suffix(s: &str) -> (&str, Option) { - if let Some(pos) = s.rfind(':') { - if s[pos + 1..].starts_with('/') { - return (&s[..pos], Some(s[pos + 1..].to_string())); - } + if let Some(pos) = s.rfind(':') + && s[pos + 1..].starts_with('/') + { + return (&s[..pos], Some(s[pos + 1..].to_string())); } (s, None) } @@ -164,20 +166,38 @@ mod tests { #[test] fn parse_fixed_delay() { let r = parse_fault_spec("delay:200ms").unwrap(); - assert!(matches!(r, FaultRule::Delay { min_ms: 200, max_ms: 200, url_pattern: None })); + assert!(matches!( + r, + FaultRule::Delay { + min_ms: 200, + max_ms: 200, + url_pattern: None + } + )); } #[test] fn parse_range_delay() { let r = parse_fault_spec("delay:100ms-500ms").unwrap(); - assert!(matches!(r, FaultRule::Delay { min_ms: 100, max_ms: 500, url_pattern: None })); + assert!(matches!( + r, + FaultRule::Delay { + min_ms: 100, + max_ms: 500, + url_pattern: None + } + )); } #[test] fn parse_delay_with_url() { let r = parse_fault_spec("delay:200ms:/api/users").unwrap(); match r { - FaultRule::Delay { min_ms: 200, max_ms: 200, url_pattern: Some(p) } => { + FaultRule::Delay { + min_ms: 200, + max_ms: 200, + url_pattern: Some(p), + } => { assert_eq!(p, "/api/users"); } _ => panic!("unexpected rule"), @@ -187,14 +207,24 @@ mod tests { #[test] fn parse_error_always() { let r = parse_fault_spec("error:503").unwrap(); - assert!(matches!(r, FaultRule::Error { status_code: 503, .. })); + assert!(matches!( + r, + FaultRule::Error { + status_code: 503, + .. + } + )); } #[test] fn parse_error_probability() { let r = parse_fault_spec("error:500:0.1").unwrap(); match r { - FaultRule::Error { status_code: 500, probability, url_pattern: None } => { + FaultRule::Error { + status_code: 500, + probability, + url_pattern: None, + } => { assert!((probability - 0.1).abs() < 1e-9); } _ => panic!("unexpected rule"), @@ -205,7 +235,11 @@ mod tests { fn parse_error_probability_and_url() { let r = parse_fault_spec("error:500:0.5:/api").unwrap(); match r { - FaultRule::Error { status_code: 500, probability, url_pattern: Some(p) } => { + FaultRule::Error { + status_code: 500, + probability, + url_pattern: Some(p), + } => { assert!((probability - 0.5).abs() < 1e-9); assert_eq!(p, "/api"); } diff --git a/crates/phantom-capture/src/lib.rs b/crates/phantom-capture/src/lib.rs index 2158019..5189e6c 100644 --- a/crates/phantom-capture/src/lib.rs +++ b/crates/phantom-capture/src/lib.rs @@ -4,7 +4,7 @@ mod proxy; #[cfg(target_os = "linux")] mod ldpreload; -pub use fault::{parse_fault_spec, FaultConfig, FaultRule}; +pub use fault::{FaultConfig, FaultRule, parse_fault_spec}; pub use proxy::ProxyCaptureBackend; #[cfg(target_os = "linux")] diff --git a/crates/phantom-capture/src/proxy.rs b/crates/phantom-capture/src/proxy.rs index 211acf3..5ae9e82 100644 --- a/crates/phantom-capture/src/proxy.rs +++ b/crates/phantom-capture/src/proxy.rs @@ -197,9 +197,7 @@ impl HttpHandler for TraceHandler { continue; } match rule { - FaultRule::Delay { - min_ms, max_ms, .. - } => { + FaultRule::Delay { min_ms, max_ms, .. } => { let delay_ms = if min_ms == max_ms { *min_ms } else { @@ -225,7 +223,15 @@ impl HttpHandler for TraceHandler { request_headers: info.request_headers, request_body: info.request_body, status_code: *status_code, - response_headers: HashMap::new(), + response_headers: { + let mut h = HashMap::new(); + h.insert( + "content-type".to_string(), + "application/json".to_string(), + ); + h.insert("x-fault-injected".to_string(), "phantom".to_string()); + h + }, response_body: Some(fault_body), timestamp: info.timestamp, duration: info.started_at.elapsed(), @@ -237,8 +243,7 @@ impl HttpHandler for TraceHandler { warn!("Trace channel full, dropping fault-injected trace"); } } - let body_bytes: bytes::Bytes = - b"{\"fault\":\"injected\"}".as_ref().into(); + let body_bytes: bytes::Bytes = b"{\"fault\":\"injected\"}".as_ref().into(); let response = Response::builder() .status(*status_code) .header("content-type", "application/json") diff --git a/src/main.rs b/src/main.rs index 0428a91..d62cf68 100644 --- a/src/main.rs +++ b/src/main.rs @@ -449,8 +449,7 @@ fn spawn_proxy_child( async fn run_proxy(cli: Cli, store: Arc) -> anyhow::Result<()> { let fault_config = build_fault_config(&cli.fault)?; - let mut backend = ProxyCaptureBackend::new(cli.port, cli.insecure) - .with_faults(fault_config); + let mut backend = ProxyCaptureBackend::new(cli.port, cli.insecure).with_faults(fault_config); let backend_name = backend.name().to_string(); let trace_rx = backend.start().map_err(|e| anyhow::anyhow!("{e}"))?; diff --git a/tests/fault_injection.rs b/tests/fault_injection.rs new file mode 100644 index 0000000..b6edbfd --- /dev/null +++ b/tests/fault_injection.rs @@ -0,0 +1,438 @@ +//! Fault injection integration tests. +//! +//! Verifies that `--fault` rules are correctly applied by the phantom proxy: +//! - `error:` returns a synthetic HTTP error without forwarding +//! - `delay:` adds measurable latency to the round-trip +//! - `error::` limits injection to matching URLs +//! +//! Requirements: `curl` on PATH (tests are skipped otherwise). +//! Run: `cargo test --test fault_injection` + +use std::io::{Read, Write as IoWrite}; +use std::net::{TcpListener, TcpStream}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── + +fn available_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind :0") + .local_addr() + .unwrap() + .port() +} + +fn wait_for_port(port: u16, timeout: Duration) -> bool { + let start = Instant::now(); + while start.elapsed() < timeout { + if TcpStream::connect(format!("127.0.0.1:{port}")).is_ok() { + return true; + } + std::thread::sleep(Duration::from_millis(50)); + } + false +} + +// ───────────────────────────────────────────────────────────────────────────── +// Mock HTTP backend +// ───────────────────────────────────────────────────────────────────────────── + +const HEALTH_BODY: &str = r#"{"status":"ok"}"#; +const USERS_BODY: &str = r#"[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]"#; + +fn route_request(req: &str) -> (&str, &str) { + let first = req.lines().next().unwrap_or(""); + if first.starts_with("GET") && first.contains("/api/health") { + ("200 OK", HEALTH_BODY) + } else if first.starts_with("GET") && first.contains("/api/users") { + ("200 OK", USERS_BODY) + } else { + ("404 Not Found", r#"{"error":"Not Found"}"#) + } +} + +fn write_response(stream: &mut impl IoWrite, status: &str, body: &str) { + let resp = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(resp.as_bytes()); + let _ = stream.flush(); +} + +fn handle_stream(stream: &mut (impl Read + IoWrite)) { + let mut buf = [0u8; 8192]; + let n = match stream.read(&mut buf) { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + let req = String::from_utf8_lossy(&buf[..n]); + let (status, body) = route_request(&req); + write_response(stream, status, body); +} + +fn start_http_backend(port: u16) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let listener = TcpListener::bind(format!("127.0.0.1:{port}")).unwrap(); + for stream in listener.incoming() { + match stream { + Ok(mut s) => handle_stream(&mut s), + Err(_) => break, + } + } + }) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Test utilities +// ───────────────────────────────────────────────────────────────────────────── + +fn parse_traces(stdout: &str) -> Vec { + stdout + .lines() + .filter(|l| l.starts_with('{')) + .filter_map(|l| serde_json::from_str(l).ok()) + .collect() +} + +/// Returns `true` if curl is not found on PATH (caller should skip the test). +fn curl_missing() -> bool { + Command::new("curl") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_err() +} + +/// Build the base curl arguments used in every fault injection test. +/// +/// Key flags: +/// --silent suppress progress meters / error messages +/// --output /dev/null discard the response body so it doesn't mix into phantom's +/// JSONL stdout; we verify the response via the trace record. +/// --proxy force curl through the phantom proxy even for localhost +/// targets (curl by default skips the proxy for 127.0.0.1). +fn curl_args(proxy_port: u16) -> Vec { + vec![ + "--silent".to_string(), + "--output".to_string(), + "/dev/null".to_string(), + "--proxy".to_string(), + format!("http://127.0.0.1:{proxy_port}"), + ] +} + +// ───────────────────────────────────────────────────────────────────────────── +// Test 1: error injection — always return a synthetic HTTP 503 +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn test_fault_error_always() { + if curl_missing() { + eprintln!("SKIP: `curl` not found"); + return; + } + + let phantom_bin = env!("CARGO_BIN_EXE_phantom"); + let tmp_dir = tempfile::tempdir().expect("tempdir"); + + let backend_port = available_port(); + let proxy_port = available_port(); + + let _backend = start_http_backend(backend_port); + assert!( + wait_for_port(backend_port, Duration::from_secs(3)), + "HTTP backend did not start" + ); + + // Run phantom with --fault error:503 and a curl child. + // no_proxy / NO_PROXY must be cleared so curl routes 127.0.0.1 through the proxy. + let out = Command::new(phantom_bin) + .args([ + "--backend", + "proxy", + "--output", + "jsonl", + "--port", + &proxy_port.to_string(), + "--data-dir", + ]) + .arg(tmp_dir.path()) + .args(["--fault", "error:503"]) + .env("no_proxy", "") + .env("NO_PROXY", "") + .arg("--") + .arg("curl") + .args(curl_args(proxy_port)) + .arg(format!("http://127.0.0.1:{backend_port}/api/health")) + .output() + .expect("run phantom"); + + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + + assert!( + out.status.success(), + "phantom exited non-zero.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + + let traces = parse_traces(&stdout); + assert_eq!( + traces.len(), + 1, + "expected 1 trace, got {}.\nstdout:\n{stdout}\nstderr:\n{stderr}", + traces.len() + ); + + let t = &traces[0]; + + // Status code must be the injected 503. + assert_eq!( + t["status_code"].as_u64(), + Some(503), + "status_code should be 503 (fault injected), trace: {t}" + ); + + // Response body must contain the fault marker. + let body = t["response_body"].as_str().unwrap_or(""); + assert!( + body.contains("fault"), + "response_body should contain 'fault', got: {body:?}" + ); + + // The x-fault-injected response header must be set. + let fault_header = t["response_headers"]["x-fault-injected"].as_str(); + assert_eq!( + fault_header, + Some("phantom"), + "x-fault-injected header should be 'phantom', got: {fault_header:?}" + ); + + // Trace and span IDs must be present. + assert!( + t["trace_id"].as_str().is_some_and(|s| !s.is_empty()), + "trace_id missing" + ); + assert!( + t["span_id"].as_str().is_some_and(|s| !s.is_empty()), + "span_id missing" + ); + + eprintln!("test_fault_error_always: OK — status=503, x-fault-injected=phantom"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Test 2: delay injection — duration_ms must be >= injected delay +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn test_fault_delay_adds_latency() { + if curl_missing() { + eprintln!("SKIP: `curl` not found"); + return; + } + + let phantom_bin = env!("CARGO_BIN_EXE_phantom"); + let tmp_dir = tempfile::tempdir().expect("tempdir"); + + let backend_port = available_port(); + let proxy_port = available_port(); + + let _backend = start_http_backend(backend_port); + assert!( + wait_for_port(backend_port, Duration::from_secs(3)), + "HTTP backend did not start" + ); + + const DELAY_MS: u64 = 300; + + let out = Command::new(phantom_bin) + .args([ + "--backend", + "proxy", + "--output", + "jsonl", + "--port", + &proxy_port.to_string(), + "--data-dir", + ]) + .arg(tmp_dir.path()) + .args(["--fault", &format!("delay:{DELAY_MS}ms")]) + .env("no_proxy", "") + .env("NO_PROXY", "") + .arg("--") + .arg("curl") + .args(curl_args(proxy_port)) + .arg(format!("http://127.0.0.1:{backend_port}/api/health")) + .output() + .expect("run phantom"); + + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + + assert!( + out.status.success(), + "phantom exited non-zero.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + + let traces = parse_traces(&stdout); + assert_eq!( + traces.len(), + 1, + "expected 1 trace, got {}.\nstdout:\n{stdout}\nstderr:\n{stderr}", + traces.len() + ); + + let t = &traces[0]; + + // The backend returned 200 — the real response was forwarded. + assert_eq!( + t["status_code"].as_u64(), + Some(200), + "status_code should be 200 (real backend response), trace: {t}" + ); + + // Duration must reflect the injected delay. + let duration_ms = t["duration_ms"].as_u64().expect("duration_ms present"); + assert!( + duration_ms >= DELAY_MS, + "duration_ms ({duration_ms}) should be >= injected delay ({DELAY_MS}ms)" + ); + + eprintln!( + "test_fault_delay_adds_latency: OK — status=200, duration={duration_ms}ms (>= {DELAY_MS}ms)" + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Test 3: URL pattern filter — only matching URLs get the fault +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn test_fault_url_pattern_filter() { + if curl_missing() { + eprintln!("SKIP: `curl` not found"); + return; + } + + let phantom_bin = env!("CARGO_BIN_EXE_phantom"); + let backend_port = available_port(); + + let _backend = start_http_backend(backend_port); + assert!( + wait_for_port(backend_port, Duration::from_secs(3)), + "HTTP backend did not start" + ); + + // Rule: only inject 503 for URLs containing "/api/health" + let fault_spec = "error:503:/api/health"; + + // ── Request 1: /api/health — should get 503 (fault injected) ──────── + let tmp1 = tempfile::tempdir().expect("tempdir"); + let proxy_port1 = available_port(); + + let out1 = Command::new(phantom_bin) + .args([ + "--backend", + "proxy", + "--output", + "jsonl", + "--port", + &proxy_port1.to_string(), + "--data-dir", + ]) + .arg(tmp1.path()) + .args(["--fault", fault_spec]) + .env("no_proxy", "") + .env("NO_PROXY", "") + .arg("--") + .arg("curl") + .args(curl_args(proxy_port1)) + .arg(format!("http://127.0.0.1:{backend_port}/api/health")) + .output() + .expect("run phantom (health)"); + + let stdout1 = String::from_utf8_lossy(&out1.stdout).into_owned(); + let stderr1 = String::from_utf8_lossy(&out1.stderr).into_owned(); + + assert!( + out1.status.success(), + "phantom exited non-zero (health).\nstdout:\n{stdout1}\nstderr:\n{stderr1}" + ); + + let traces1 = parse_traces(&stdout1); + assert_eq!( + traces1.len(), + 1, + "expected 1 trace for /api/health, got {}.\nstdout:\n{stdout1}", + traces1.len() + ); + assert_eq!( + traces1[0]["status_code"].as_u64(), + Some(503), + "/api/health should be 503 (fault injected), trace: {}", + traces1[0] + ); + + // ── Request 2: /api/users — should get 200 (no fault) ─────────────── + let tmp2 = tempfile::tempdir().expect("tempdir"); + let proxy_port2 = available_port(); + + let out2 = Command::new(phantom_bin) + .args([ + "--backend", + "proxy", + "--output", + "jsonl", + "--port", + &proxy_port2.to_string(), + "--data-dir", + ]) + .arg(tmp2.path()) + .args(["--fault", fault_spec]) + .env("no_proxy", "") + .env("NO_PROXY", "") + .arg("--") + .arg("curl") + .args(curl_args(proxy_port2)) + .arg(format!("http://127.0.0.1:{backend_port}/api/users")) + .output() + .expect("run phantom (users)"); + + let stdout2 = String::from_utf8_lossy(&out2.stdout).into_owned(); + let stderr2 = String::from_utf8_lossy(&out2.stderr).into_owned(); + + assert!( + out2.status.success(), + "phantom exited non-zero (users).\nstdout:\n{stdout2}\nstderr:\n{stderr2}" + ); + + let traces2 = parse_traces(&stdout2); + assert_eq!( + traces2.len(), + 1, + "expected 1 trace for /api/users, got {}.\nstdout:\n{stdout2}", + traces2.len() + ); + assert_eq!( + traces2[0]["status_code"].as_u64(), + Some(200), + "/api/users should be 200 (fault pattern did not match), trace: {}", + traces2[0] + ); + + // Verify the real backend response body was forwarded. + let users_body = traces2[0]["response_body"].as_str().unwrap_or(""); + assert!( + users_body.contains("Alice"), + "/api/users response should contain 'Alice', got: {users_body:?}" + ); + + eprintln!( + "test_fault_url_pattern_filter: OK — /api/health=503 (injected), /api/users=200 (passthrough)" + ); +}