From e8e861a326829ffa72c3d8e7db5367db272a4ddb Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Sun, 29 Mar 2026 22:48:15 -0700 Subject: [PATCH 01/33] WIP on wound-wait for 2pc --- crates/client-api/src/routes/database.rs | 51 ++- crates/core/DISTRIBUTED-WOUND-WAIT.md | 143 +++++++ crates/core/src/client/client_connection.rs | 2 + crates/core/src/client/messages.rs | 4 +- crates/core/src/error.rs | 2 + crates/core/src/host/global_tx.rs | 306 +++++++++++++++ crates/core/src/host/host_controller.rs | 5 + crates/core/src/host/instance_env.rs | 68 +++- crates/core/src/host/mod.rs | 1 + crates/core/src/host/module_host.rs | 370 +++++++++++++++++- crates/core/src/host/v8/mod.rs | 20 +- crates/core/src/host/v8/syscall/v1.rs | 1 + crates/core/src/host/v8/syscall/v2.rs | 1 + crates/core/src/host/wasm_common.rs | 1 + .../src/host/wasm_common/module_host_actor.rs | 100 ++++- .../src/host/wasmtime/wasm_instance_env.rs | 17 +- .../core/src/host/wasmtime/wasmtime_module.rs | 8 +- crates/core/src/replica_context.rs | 6 + .../subscription/module_subscription_actor.rs | 6 +- crates/datastore/src/execution_context.rs | 1 - crates/lib/src/lib.rs | 2 + crates/lib/src/tx_id.rs | 64 +++ 22 files changed, 1145 insertions(+), 34 deletions(-) create mode 100644 crates/core/DISTRIBUTED-WOUND-WAIT.md create mode 100644 crates/core/src/host/global_tx.rs create mode 100644 crates/lib/src/tx_id.rs diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 5077d6756cc..f0f4fa9da9c 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -20,6 +20,7 @@ use axum::response::{ErrorResponse, IntoResponse}; use axum::routing::MethodRouter; use axum::Extension; use axum_extra::TypedHeader; +use http::HeaderMap; use futures::TryStreamExt; use http::StatusCode; use log::{info, warn}; @@ -41,7 +42,7 @@ use spacetimedb_lib::bsatn; use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::DeserializeSeed; -use spacetimedb_lib::{sats, AlgebraicValue, Hash, ProductValue, Timestamp}; +use spacetimedb_lib::{sats, AlgebraicValue, GlobalTxId, Hash, ProductValue, Timestamp, TX_ID_HEADER}; use spacetimedb_schema::auto_migrate::{ MigrationPolicy as SchemaMigrationPolicy, MigrationToken, PrettyPrintStyle as AutoMigratePrettyPrintStyle, }; @@ -133,6 +134,7 @@ fn map_procedure_error(e: ProcedureCallError, procedure: &str) -> (StatusCode, S pub async fn call( State(worker_ctx): State, Extension(auth): Extension, + headers: HeaderMap, Path(CallParams { name_or_identity, reducer, @@ -141,6 +143,10 @@ pub async fn call( body: Bytes, ) -> axum::response::Result { let caller_identity = auth.claims.identity; + let tx_id = headers + .get(TX_ID_HEADER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()); let args = parse_call_args(content_type, body)?; @@ -162,6 +168,7 @@ pub async fn call( .call_reducer_with_return( caller_identity, Some(connection_id), + tx_id, None, None, None, @@ -251,6 +258,7 @@ fn parse_call_args(content_type: headers::ContentType, body: Bytes) -> axum::res pub async fn prepare( State(worker_ctx): State, Extension(auth): Extension, + headers: HeaderMap, Path(CallParams { name_or_identity, reducer, @@ -260,6 +268,10 @@ pub async fn prepare( ) -> axum::response::Result { let args = parse_call_args(content_type, body)?; let caller_identity = auth.claims.identity; + let tx_id = headers + .get(TX_ID_HEADER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()); let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; @@ -267,7 +279,7 @@ pub async fn prepare( // call_identity_connected/disconnected submit jobs to the module's executor, which // will be blocked holding the 2PC write lock after prepare_reducer returns — deadlock. let result = module - .prepare_reducer(caller_identity, None, &reducer, args) + .prepare_reducer(caller_identity, None, tx_id, &reducer, args) .await; match result { @@ -298,6 +310,12 @@ pub struct TwoPcParams { prepare_id: String, } +#[derive(Deserialize)] +pub struct GlobalTxParams { + name_or_identity: NameOrIdentity, + global_tx_id: String, +} + /// 2PC commit endpoint: finalize a prepared transaction. /// /// `POST /v1/database/:name_or_identity/2pc/commit/:prepare_id` @@ -389,6 +407,30 @@ pub async fn ack_commit_2pc( Ok(StatusCode::OK) } +/// 2PC wound endpoint. +/// +/// `POST /v1/database/:name_or_identity/2pc/wound/:global_tx_id` +pub async fn wound_2pc( + State(worker_ctx): State, + Extension(_auth): Extension, + Path(GlobalTxParams { + name_or_identity, + global_tx_id, + }): Path, +) -> axum::response::Result { + let tx_id = global_tx_id + .parse::() + .map_err(|e| (StatusCode::BAD_REQUEST, e).into_response())?; + let (module, _database) = find_module_and_database(&worker_ctx, name_or_identity).await?; + + module.wound_global_tx(tx_id).await.map_err(|e| { + log::warn!("2PC wound failed for {tx_id}: {e}"); + (StatusCode::NOT_FOUND, e).into_response() + })?; + + Ok(StatusCode::OK) +} + fn reducer_outcome_response( module: &ModuleHost, owner_identity: &Identity, @@ -426,6 +468,7 @@ fn reducer_outcome_response( // TODO: different status code? this is what cloudflare uses, sorta Ok((StatusCode::from_u16(530).unwrap(), (*errmsg).into_response())) } + ReducerOutcome::Wounded(errmsg) => Ok((StatusCode::CONFLICT, (*errmsg).into_response())), ReducerOutcome::BudgetExceeded => { log::warn!("Node's energy budget exceeded for identity: {owner_identity} while executing {reducer}"); Ok(( @@ -1401,6 +1444,8 @@ pub struct DatabaseRoutes { pub commit_2pc_post: MethodRouter, /// POST: /database/:name_or_identity/2pc/abort/:prepare_id pub abort_2pc_post: MethodRouter, + /// POST: /database/:name_or_identity/2pc/wound/:global_tx_id + pub wound_2pc_post: MethodRouter, /// GET: /database/:name_or_identity/2pc/status/:prepare_id pub status_2pc_get: MethodRouter, /// POST: /database/:name_or_identity/2pc/ack-commit/:prepare_id @@ -1433,6 +1478,7 @@ where prepare_post: post(prepare::), commit_2pc_post: post(commit_2pc::), abort_2pc_post: post(abort_2pc::), + wound_2pc_post: post(wound_2pc::), status_2pc_get: get(status_2pc::), ack_commit_2pc_post: post(ack_commit_2pc::), } @@ -1463,6 +1509,7 @@ where .route("/prepare/:reducer", self.prepare_post) .route("/2pc/commit/:prepare_id", self.commit_2pc_post) .route("/2pc/abort/:prepare_id", self.abort_2pc_post) + .route("/2pc/wound/:global_tx_id", self.wound_2pc_post) .route("/2pc/status/:prepare_id", self.status_2pc_get) .route("/2pc/ack-commit/:prepare_id", self.ack_commit_2pc_post); diff --git a/crates/core/DISTRIBUTED-WOUND-WAIT.md b/crates/core/DISTRIBUTED-WOUND-WAIT.md new file mode 100644 index 00000000000..750c36a3dee --- /dev/null +++ b/crates/core/DISTRIBUTED-WOUND-WAIT.md @@ -0,0 +1,143 @@ +# Distributed Wound-Wait for 2PC Deadlock Breaking + +## Problem + +Distributed reducers can deadlock when one distributed transaction holds a participant lock on one database and waits on another database locked by a younger distributed transaction. Existing 2PC ensures atomic commit/abort, but it does not resolve distributed lock cycles. + +## Chosen Model + +- Use wound-wait. +- Transaction identity is `GlobalTxId`. +- `GlobalTxId.creator_db` is the authoritative coordinator database. +- Participant and coordinator runtime state are keyed by `GlobalTxId`. +- `prepare_id` remains a participant-local 2PC phase handle only. +- Older transactions wound younger transactions. +- Younger transactions wait behind older transactions. +- Lock acquisition is managed by an owner-aware scheduler that tracks running and pending `GlobalTxId`s. +- A wound RPC is required because a younger lock holder may belong to a distributed transaction coordinated on a different database. + +## Runtime Model + +- Add a distributed session registry keyed by `GlobalTxId`. +- Session tracks role, state, local `prepare_id`, coordinator identity, participants, and a local wounded/abort signal. +- Add a per-database async lock scheduler for distributed reducer write acquisition. +- Scheduler state includes current running owner, pending queue/set, and wounded marker for the current owner. +- Requesters await scheduler admission before reducer execution starts a local mutable transaction. + +## Wound Protocol + +Participant detecting conflict compares requester and owner by `GlobalTxId` ordering. + +- If requester is older: + - mark local owner as wounded + - send wound RPC to coordinator `GlobalTxId.creator_db` if needed + - keep requester pending until local owner releases +- If requester is younger: + - requester stays pending + +Wound RPC is idempotent and targets the distributed session at the coordinator. + +Coordinator receiving wound: + +- transitions session to `Aborting` +- sets wounded flag +- aborts local execution cooperatively +- fans out abort to known prepared participants + +## Safe Points + +- Before remote reducer calls +- Before PREPARE / COMMIT path work +- After reducer body returns, before expensive post-processing + +On safe-point wound detection: + +- rollback local tx +- unregister scheduler ownership +- wake waiters +- surface retryable `wounded` error + +## Compatibility + +- Keep existing `/2pc/commit`, `/2pc/abort`, `/2pc/status`, and ack-commit flows. +- Add a new wound endpoint. +- `/prepare` must propagate `GlobalTxId` the same way `/call` already does. +- No durable format change unless recovery work later proves it necessary. + +## Implementation Sequence + +### 1. Propagate `GlobalTxId` through 2PC prepare path + +- Update outgoing 2PC prepare requests to send `X-Spacetime-Tx-Id`. +- Update incoming `/prepare/:reducer` to parse `X-Spacetime-Tx-Id`. +- Thread `GlobalTxId` through `prepare_reducer` and any participant execution params. +- Ensure recovered/replayed participant work can recover or reconstruct the same session identity. + +### 2. Replace minimal prepared registry with `GlobalTxId` session registry + +- Extend the current prepared transaction registry into a session manager keyed by `GlobalTxId`. +- Track: + - role + - state + - local `prepare_id` + - participants + - coordinator identity + - wounded/abort signal +- Provide lookup by both `GlobalTxId` and `prepare_id`. + +### 3. Add distributed lock scheduler + +- Add an async scheduler in `core`, adjacent to reducer tx startup, not inside raw datastore locking. +- Track running owner and pending `GlobalTxId`s. +- Require distributed reducer write acquisition to await scheduler admission before blocking datastore acquisition. +- Implement wound-wait ordering and wakeup behavior there. + +### 4. Add wound RPC endpoint and coordinator handler + +- Add `POST /v1/database/:name_or_identity/2pc/wound/:global_tx_id`. +- Parse and route by `GlobalTxId.creator_db`. +- Coordinator session handler must: + - mark session `Aborting` + - set wounded flag + - begin participant abort fanout + - behave idempotently + +### 5. Add cooperative abort checks in reducer execution + +- Add wound checks at required safe points. +- On wound: + - rollback + - unregister running owner + - notify scheduler waiters + - surface retryable wounded error + +### 6. Integrate scheduler + wound with 2PC transitions + +- Ensure PREPARE, COMMIT, ABORT, and recovery all keep scheduler and session registry consistent. +- Make local owner release happen on all terminal paths. +- Keep participant recovery compatible with session state. + +### 7. Add tests + +- Scheduler ordering and wakeup tests +- Local same-database wound tests +- Distributed deadlock cycle tests +- Wound RPC idempotency tests +- Recovery tests for wounded prepared transactions +- Regression tests for existing 2PC success/failure flows + +## Acceptance Criteria + +- Distributed deadlock cycles are broken deterministically by wound-wait. +- Older distributed transactions eventually proceed without manual intervention. +- Younger distributed transactions abort globally, not just locally. +- `/prepare` and `/call` both carry `GlobalTxId`. +- Existing 2PC happy paths continue to pass. +- Repeated wound or abort requests are safe and idempotent. + +## Assumptions + +- `GlobalTxId.creator_db` is always the coordinator database. +- `GlobalTxId` ordering is the authoritative age/tie-break rule. +- Cooperative abort at safe points is sufficient for v1; no preemptive interruption is required. +- Lock scheduler state is in-memory runtime state, not durable state. diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 6fb8d8e1623..21ff74ee822 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -852,6 +852,7 @@ impl ClientConnection { .call_reducer( self.id.identity, Some(self.id.connection_id), + None, caller, Some(request_id), Some(timer), @@ -873,6 +874,7 @@ impl ClientConnection { .call_reducer( self.id.identity, Some(self.id.connection_id), + None, Some(self.sender()), Some(request_id), Some(timer), diff --git a/crates/core/src/client/messages.rs b/crates/core/src/client/messages.rs index ed65e092d0e..18e88ddc3c5 100644 --- a/crates/core/src/client/messages.rs +++ b/crates/core/src/client/messages.rs @@ -379,7 +379,9 @@ impl ToProtocol for TransactionUpdateMessage { let status = match &event.status { EventStatus::Committed(_) => ws_v1::UpdateStatus::Committed(update), - EventStatus::FailedUser(errmsg) | EventStatus::FailedInternal(errmsg) => { + EventStatus::FailedUser(errmsg) + | EventStatus::FailedInternal(errmsg) + | EventStatus::Wounded(errmsg) => { ws_v1::UpdateStatus::Failed(errmsg.clone().into()) } EventStatus::OutOfEnergy => ws_v1::UpdateStatus::OutOfEnergy, diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 226c2700d08..4b0f85bf70d 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -275,6 +275,8 @@ pub enum NodesError { BadIndexType(u8), #[error("Failed to scheduled timer: {0}")] ScheduleError(#[source] ScheduleError), + #[error("Distributed transaction wounded: {0}")] + Wounded(String), #[error("HTTP request failed: {0}")] HttpError(String), } diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs new file mode 100644 index 00000000000..bf478ef7c80 --- /dev/null +++ b/crates/core/src/host/global_tx.rs @@ -0,0 +1,306 @@ +use crate::identity::Identity; +use spacetimedb_lib::GlobalTxId; +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::sync::Notify; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GlobalTxRole { + Coordinator, + Participant, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GlobalTxState { + Running, + Preparing, + Prepared, + Aborting, + Aborted, + Committing, + Committed, +} + +#[derive(Debug)] +pub struct GlobalTxSession { + pub tx_id: GlobalTxId, + pub role: GlobalTxRole, + pub coordinator_identity: Identity, + wounded: AtomicBool, + state: Mutex, + prepare_id: Mutex>, + participants: Mutex>, +} + +impl GlobalTxSession { + fn new(tx_id: GlobalTxId, role: GlobalTxRole, coordinator_identity: Identity) -> Self { + Self { + tx_id, + role, + coordinator_identity, + wounded: AtomicBool::new(false), + state: Mutex::new(GlobalTxState::Running), + prepare_id: Mutex::new(None), + participants: Mutex::new(HashMap::new()), + } + } + + pub fn is_wounded(&self) -> bool { + self.wounded.load(Ordering::SeqCst) + } + + pub fn wound(&self) -> bool { + !self.wounded.swap(true, Ordering::SeqCst) + } + + pub fn state(&self) -> GlobalTxState { + *self.state.lock().unwrap() + } + + pub fn set_state(&self, state: GlobalTxState) { + *self.state.lock().unwrap() = state; + } + + pub fn set_prepare_id(&self, prepare_id: Option) { + *self.prepare_id.lock().unwrap() = prepare_id; + } + + pub fn prepare_id(&self) -> Option { + self.prepare_id.lock().unwrap().clone() + } + + pub fn add_participant(&self, db_identity: Identity, prepare_id: String) { + self.participants.lock().unwrap().insert(db_identity, prepare_id); + } + + pub fn participants(&self) -> Vec<(Identity, String)> { + self.participants + .lock() + .unwrap() + .iter() + .map(|(db, pid)| (*db, pid.clone())) + .collect() + } +} + +struct LockState { + owner: Option, + waiting: HashSet, + wounded_owners: HashSet, +} + +impl Default for LockState { + fn default() -> Self { + Self { + owner: None, + waiting: HashSet::new(), + wounded_owners: HashSet::new(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AcquireDisposition { + Acquired, + Wound(GlobalTxId), + Wait, +} + +#[derive(Default)] +pub struct GlobalTxManager { + sessions: Mutex>>, + prepare_to_tx: Mutex>, + lock_state: Mutex, + lock_notify: Notify, +} + +impl GlobalTxManager { + pub fn ensure_session( + &self, + tx_id: GlobalTxId, + role: GlobalTxRole, + coordinator_identity: Identity, + ) -> Arc { + let mut sessions = self.sessions.lock().unwrap(); + sessions + .entry(tx_id) + .or_insert_with(|| Arc::new(GlobalTxSession::new(tx_id, role, coordinator_identity))) + .clone() + } + + pub fn get_session(&self, tx_id: &GlobalTxId) -> Option> { + self.sessions.lock().unwrap().get(tx_id).cloned() + } + + pub fn remove_session(&self, tx_id: &GlobalTxId) { + self.sessions.lock().unwrap().remove(tx_id); + } + + pub fn tx_for_prepare(&self, prepare_id: &str) -> Option { + self.prepare_to_tx.lock().unwrap().get(prepare_id).copied() + } + + pub fn set_prepare_mapping(&self, tx_id: GlobalTxId, prepare_id: String) { + self.prepare_to_tx.lock().unwrap().insert(prepare_id.clone(), tx_id); + if let Some(session) = self.get_session(&tx_id) { + session.set_prepare_id(Some(prepare_id)); + } + } + + pub fn remove_prepare_mapping(&self, prepare_id: &str) -> Option { + let tx_id = self.prepare_to_tx.lock().unwrap().remove(prepare_id); + if let Some(tx_id) = tx_id + && let Some(session) = self.get_session(&tx_id) + { + session.set_prepare_id(None); + } + tx_id + } + + pub fn add_participant(&self, tx_id: GlobalTxId, db_identity: Identity, prepare_id: String) { + if let Some(session) = self.get_session(&tx_id) { + session.add_participant(db_identity, prepare_id); + } + } + + pub fn mark_state(&self, tx_id: &GlobalTxId, state: GlobalTxState) { + if let Some(session) = self.get_session(tx_id) { + session.set_state(state); + } + } + + pub fn is_wounded(&self, tx_id: &GlobalTxId) -> bool { + self.get_session(tx_id).map(|s| s.is_wounded()).unwrap_or(false) + } + + pub fn wound(&self, tx_id: &GlobalTxId) -> Option> { + let session = self.get_session(tx_id)?; + let _ = session.wound(); + if !matches!(session.state(), GlobalTxState::Committed | GlobalTxState::Aborted) { + session.set_state(GlobalTxState::Aborting); + } + Some(session) + } + + pub async fn acquire(&self, tx_id: GlobalTxId) -> AcquireDisposition { + loop { + let waiter = { + let mut state = self.lock_state.lock().unwrap(); + match state.owner { + None => { + state.owner = Some(tx_id); + state.waiting.remove(&tx_id); + return AcquireDisposition::Acquired; + } + Some(owner) if owner == tx_id => { + state.waiting.remove(&tx_id); + return AcquireDisposition::Acquired; + } + Some(owner) => { + state.waiting.insert(tx_id); + if tx_id < owner && state.wounded_owners.insert(owner) { + return AcquireDisposition::Wound(owner); + } + self.lock_notify.notified() + } + } + }; + waiter.await; + } + } + + pub fn release(&self, tx_id: &GlobalTxId) { + let mut state = self.lock_state.lock().unwrap(); + if state.owner.as_ref() == Some(tx_id) { + state.owner = None; + state.wounded_owners.remove(tx_id); + self.lock_notify.notify_waiters(); + } + state.waiting.remove(tx_id); + } +} + +#[cfg(test)] +mod tests { + use super::{AcquireDisposition, GlobalTxManager}; + use crate::identity::Identity; + use spacetimedb_lib::{GlobalTxId, Timestamp}; + use std::sync::Arc; + use tokio::runtime::Runtime; + use std::time::Duration; + + fn tx_id(ts: i64, db_byte: u8, nonce: u32) -> GlobalTxId { + GlobalTxId::new( + Timestamp::from_micros_since_unix_epoch(ts), + Identity::from_byte_array([db_byte; 32]), + nonce, + ) + } + + #[test] + fn older_requester_wounds_younger_owner() { + let manager = GlobalTxManager::default(); + let younger = tx_id(20, 2, 0); + let older = tx_id(10, 1, 0); + manager.ensure_session( + younger, + super::GlobalTxRole::Participant, + younger.creator_db, + ); + + let rt = Runtime::new().unwrap(); + assert_eq!(rt.block_on(manager.acquire(younger)), AcquireDisposition::Acquired); + assert_eq!(rt.block_on(manager.acquire(older)), AcquireDisposition::Wound(younger)); + assert!(manager.wound(&younger).is_some()); + assert!(manager.is_wounded(&younger)); + } + + #[test] + fn younger_requester_waits_behind_older_owner() { + let manager = GlobalTxManager::default(); + let older = tx_id(10, 1, 0); + let younger = tx_id(20, 2, 0); + let rt = Runtime::new().unwrap(); + + assert_eq!(rt.block_on(manager.acquire(older)), AcquireDisposition::Acquired); + let wait = rt.block_on(async { + tokio::time::timeout(Duration::from_millis(25), manager.acquire(younger)).await + }); + assert!(wait.is_err()); + } + + #[test] + fn waiter_acquires_after_release() { + let manager = Arc::new(GlobalTxManager::default()); + let owner = tx_id(10, 1, 0); + let waiter = tx_id(20, 2, 0); + let rt = Runtime::new().unwrap(); + + assert_eq!(rt.block_on(manager.acquire(owner)), AcquireDisposition::Acquired); + + let manager_for_thread = manager.clone(); + let handle = std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + assert_eq!(rt.block_on(manager_for_thread.acquire(waiter)), AcquireDisposition::Acquired); + manager_for_thread.release(&waiter); + }); + + std::thread::sleep(Duration::from_millis(25)); + manager.release(&owner); + handle.join().unwrap(); + } + + #[test] + fn wound_is_idempotent() { + let manager = GlobalTxManager::default(); + let tx_id = tx_id(10, 1, 0); + let session = manager.ensure_session(tx_id, super::GlobalTxRole::Coordinator, tx_id.creator_db); + + assert!(!session.is_wounded()); + assert!(manager.wound(&tx_id).is_some()); + assert!(session.is_wounded()); + assert!(manager.wound(&tx_id).is_some()); + assert!(session.is_wounded()); + } +} diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 8ab53513314..bca5278486b 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -180,6 +180,7 @@ impl From for Result<(), anyhow::Error> { pub enum ReducerOutcome { Committed, Failed(Box>), + Wounded(Box>), BudgetExceeded, } @@ -188,6 +189,7 @@ impl ReducerOutcome { match self { Self::Committed => Ok(()), Self::Failed(e) => Err(anyhow::anyhow!(e)), + Self::Wounded(e) => Err(anyhow::anyhow!(e)), Self::BudgetExceeded => Err(anyhow::anyhow!("reducer ran out of energy")), } } @@ -204,6 +206,7 @@ impl From<&EventStatus> for ReducerOutcome { EventStatus::FailedUser(e) | EventStatus::FailedInternal(e) => { ReducerOutcome::Failed(Box::new((&**e).into())) } + EventStatus::Wounded(e) => ReducerOutcome::Wounded(Box::new((&**e).into())), EventStatus::OutOfEnergy => ReducerOutcome::BudgetExceeded, } } @@ -727,6 +730,8 @@ async fn make_replica_ctx( call_reducer_client, call_reducer_router, call_reducer_auth_token, + tx_id_nonce: Arc::default(), + global_tx_manager: Arc::default(), }) } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index a211b5603fc..75eb421a354 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -2,6 +2,7 @@ use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; use crate::error::{DBError, DatastoreError, IndexError, NodesError}; +use crate::host::global_tx::{GlobalTxRole, GlobalTxState}; use crate::host::module_host::{DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall}; use crate::host::wasm_common::TimingSpan; use crate::replica_context::ReplicaContext; @@ -20,7 +21,7 @@ use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId}; use spacetimedb_datastore::traits::IsolationLevel; -use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp}; +use spacetimedb_lib::{http as st_http, ConnectionId, GlobalTxId, Identity, Timestamp, TX_ID_HEADER}; use spacetimedb_primitives::{ColId, ColList, IndexId, TableId}; use spacetimedb_sats::{ bsatn::{self, ToBsatn}, @@ -34,6 +35,7 @@ use std::fmt::Display; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::ops::DerefMut; +use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::{Duration, Instant}; use std::vec::IntoIter; @@ -50,6 +52,8 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Distributed transaction id for the current reducer invocation. + current_tx_id: Option, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -241,6 +245,7 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + current_tx_id: None, in_anon_tx: false, procedure_last_tx_offset: None, prepared_participants: Vec::new(), @@ -253,11 +258,32 @@ impl InstanceEnv { } /// Signal to this `InstanceEnv` that a function call is beginning. - pub fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType) { + pub fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType, tx_id: Option) { + let is_reducer = matches!(func_type, FuncCallType::Reducer); self.start_time = ts; self.start_instant = Instant::now(); self.func_type = func_type; self.func_name = Some(name); + self.current_tx_id = if is_reducer { + Some(tx_id.unwrap_or_else(|| self.mint_tx_id(ts))) + } else { + None + }; + } + + pub fn current_tx_id(&self) -> Option { + self.current_tx_id + } + + fn wounded_tx_error(&self, tx_id: GlobalTxId) -> NodesError { + NodesError::Wounded(format!( + "distributed transaction {tx_id} was wounded by an older transaction" + )) + } + + fn mint_tx_id(&self, start_ts: Timestamp) -> GlobalTxId { + let nonce = self.replica_ctx.tx_id_nonce.fetch_add(1, Ordering::Relaxed); + GlobalTxId::new(start_ts, self.replica_ctx.database.database_identity, nonce) } /// Returns the name of the most recent reducer to be run in this environment, @@ -1010,8 +1036,19 @@ impl InstanceEnv { // accepts the request without generating a fresh ephemeral identity per call. let auth_token = self.replica_ctx.call_reducer_auth_token.clone(); let caller_identity = self.replica_ctx.database.database_identity; + let tx_id = self.current_tx_id(); + let wounded_error = tx_id.and_then(|tx_id| { + self.replica_ctx + .global_tx_manager + .is_wounded(&tx_id) + .then(|| self.wounded_tx_error(tx_id)) + }); async move { + if let Some(err) = wounded_error { + return Err(err); + } + let start = Instant::now(); let base_url = router @@ -1031,6 +1068,9 @@ impl InstanceEnv { if let Some(token) = auth_token { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } + if let Some(tx_id) = tx_id { + req = req.header(TX_ID_HEADER, tx_id.to_string()); + } let result = async { let response = req.send().await.map_err(|e| NodesError::HttpError(e.to_string()))?; let status = response.status().as_u16(); @@ -1076,8 +1116,27 @@ impl InstanceEnv { let reducer_name = reducer_name.to_owned(); let auth_token = self.replica_ctx.call_reducer_auth_token.clone(); let caller_identity = self.replica_ctx.database.database_identity; + let tx_id = self.current_tx_id(); + let wounded_error = tx_id.and_then(|tx_id| { + self.replica_ctx + .global_tx_manager + .is_wounded(&tx_id) + .then(|| self.wounded_tx_error(tx_id)) + }); + + if let Some(tx_id) = tx_id { + let session = self + .replica_ctx + .global_tx_manager + .ensure_session(tx_id, GlobalTxRole::Coordinator, tx_id.creator_db); + session.set_state(GlobalTxState::Preparing); + } async move { + if let Some(err) = wounded_error { + return Err(err); + } + let start = Instant::now(); let base_url = router @@ -1097,6 +1156,9 @@ impl InstanceEnv { if let Some(token) = auth_token { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } + if let Some(tx_id) = tx_id { + req = req.header(TX_ID_HEADER, tx_id.to_string()); + } let result = async { let response = req.send().await.map_err(|e| NodesError::HttpError(e.to_string()))?; let status = response.status().as_u16(); @@ -1502,6 +1564,8 @@ mod test { call_reducer_client: ReplicaContext::new_call_reducer_client(&CallReducerOnDbConfig::default()), call_reducer_router: Arc::new(LocalReducerRouter::new("http://127.0.0.1:3000")), call_reducer_auth_token: None, + tx_id_nonce: Arc::default(), + global_tx_manager: Arc::default(), }, runtime, )) diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 06e55de6444..2213206138b 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -37,6 +37,7 @@ where } mod disk_storage; +pub mod global_tx; mod host_controller; mod module_common; #[allow(clippy::too_many_arguments)] diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index ab0115dc986..b984e695e37 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -55,7 +55,7 @@ use spacetimedb_expr::expr::CollectViews; use spacetimedb_lib::db::raw_def::v9::Lifecycle; use spacetimedb_lib::identity::{AuthCtx, RequestId}; use spacetimedb_lib::metrics::ExecutionMetrics; -use spacetimedb_lib::{ConnectionId, Timestamp}; +use spacetimedb_lib::{ConnectionId, GlobalTxId, Timestamp}; use spacetimedb_primitives::{ArgId, ProcedureId, TableId, ViewFnPtr, ViewId}; use spacetimedb_query::compile_subscription; use spacetimedb_sats::raw_identifier::RawIdentifier; @@ -170,6 +170,7 @@ pub enum EventStatus { Committed(DatabaseUpdate), FailedUser(String), FailedInternal(String), + Wounded(String), OutOfEnergy, } @@ -231,6 +232,32 @@ pub struct ModuleInfo { pub metrics: ModuleMetrics, } +struct GlobalTxAdmissionGuard<'a> { + module_host: &'a ModuleHost, + tx_id: Option, +} + +impl<'a> GlobalTxAdmissionGuard<'a> { + fn new(module_host: &'a ModuleHost, tx_id: GlobalTxId) -> Self { + Self { + module_host, + tx_id: Some(tx_id), + } + } + + fn disarm(mut self) { + self.tx_id = None; + } +} + +impl Drop for GlobalTxAdmissionGuard<'_> { + fn drop(&mut self) { + if let Some(tx_id) = self.tx_id.take() { + self.module_host.abort_global_tx_locally(tx_id, true); + } + } +} + impl fmt::Debug for ModuleInfo { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ModuleInfo") @@ -570,6 +597,7 @@ pub fn call_identity_connected( None, None, None, + None, reducer_id, reducer_def, FunctionArgs::Nullary, @@ -594,6 +622,7 @@ pub fn call_identity_connected( // If the reducer returned an error or couldn't run due to insufficient energy, // abort the connection: the module code has decided it doesn't want this client. ReducerOutcome::Failed(message) => Err(ClientConnectedError::Rejected(*message)), + ReducerOutcome::Wounded(message) => Err(ClientConnectedError::Rejected(*message)), ReducerOutcome::BudgetExceeded => Err(ClientConnectedError::OutOfEnergy), } } else { @@ -619,6 +648,7 @@ pub struct CallReducerParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub tx_id: Option, pub client: Option>, pub request_id: Option, pub timer: Option, @@ -639,6 +669,7 @@ impl CallReducerParams { timestamp, caller_identity, caller_connection_id: ConnectionId::ZERO, + tx_id: None, client: None, request_id: None, timer: None, @@ -963,7 +994,9 @@ impl From for ViewOutcome { fn from(status: EventStatus) -> Self { match status { EventStatus::Committed(_) => ViewOutcome::Success, - EventStatus::FailedUser(e) | EventStatus::FailedInternal(e) => ViewOutcome::Failed(e), + EventStatus::FailedUser(e) | EventStatus::FailedInternal(e) | EventStatus::Wounded(e) => { + ViewOutcome::Failed(e) + } EventStatus::OutOfEnergy => ViewOutcome::BudgetExceeded, } } @@ -1468,6 +1501,7 @@ impl ModuleHost { None, None, None, + None, reducer_id, reducer_def, FunctionArgs::Nullary, @@ -1491,7 +1525,7 @@ impl ModuleHost { fallback() } Ok(ReducerCallResult { - outcome: ReducerOutcome::Failed(_) | ReducerOutcome::BudgetExceeded, + outcome: ReducerOutcome::Failed(_) | ReducerOutcome::Wounded(_) | ReducerOutcome::BudgetExceeded, .. }) => fallback(), @@ -1552,6 +1586,7 @@ impl ModuleHost { module: &ModuleInfo, caller_identity: Identity, caller_connection_id: Option, + tx_id: Option, client: Option>, request_id: Option, timer: Option, @@ -1567,6 +1602,7 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + tx_id, client, request_id, timer, @@ -1579,6 +1615,7 @@ impl ModuleHost { &self, caller_identity: Identity, caller_connection_id: Option, + tx_id: Option, client: Option>, request_id: Option, timer: Option, @@ -1589,11 +1626,26 @@ impl ModuleHost { let args = args .into_tuple_for_def(&self.info.module_def, reducer_def) .map_err(InvalidReducerArguments)?; + let admission_guard = if let Some(tx_id) = tx_id { + match self.acquire_global_tx_slot(tx_id).await { + Ok(guard) => Some(guard), + Err(outcome) => { + return Ok(ReducerCallResult { + outcome, + energy_used: EnergyQuanta::ZERO, + execution_duration: Default::default(), + }) + } + } + } else { + None + }; let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); let call_reducer_params = CallReducerParams { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + tx_id, client, request_id, timer, @@ -1601,19 +1653,24 @@ impl ModuleHost { args, }; - self.call( + let result = self.call( &reducer_def.name, call_reducer_params, async |p, inst| Ok(inst.call_reducer(p)), async |p, inst| inst.call_reducer(p).await, ) - .await? + .await; + if let Some(guard) = admission_guard { + guard.disarm(); + } + result? } async fn call_reducer_inner_with_return( &self, caller_identity: Identity, caller_connection_id: Option, + tx_id: Option, client: Option>, request_id: Option, timer: Option, @@ -1624,11 +1681,29 @@ impl ModuleHost { let args = args .into_tuple_for_def(&self.info.module_def, reducer_def) .map_err(InvalidReducerArguments)?; + let admission_guard = if let Some(tx_id) = tx_id { + match self.acquire_global_tx_slot(tx_id).await { + Ok(guard) => Some(guard), + Err(outcome) => { + return Ok(( + ReducerCallResult { + outcome, + energy_used: EnergyQuanta::ZERO, + execution_duration: Default::default(), + }, + None, + )) + } + } + } else { + None + }; let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); let call_reducer_params = CallReducerParams { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + tx_id, client, request_id, timer, @@ -1636,19 +1711,24 @@ impl ModuleHost { args, }; - self.call( + let result = self.call( &reducer_def.name, call_reducer_params, async |p, inst| Ok(inst.call_reducer_with_return(p)), async |p, inst| inst.call_reducer(p).await.map(|res| (res, None)), ) - .await? + .await; + if let Some(guard) = admission_guard { + guard.disarm(); + } + result? } pub async fn call_reducer( &self, caller_identity: Identity, caller_connection_id: Option, + tx_id: Option, client: Option>, request_id: Option, timer: Option, @@ -1672,6 +1752,7 @@ impl ModuleHost { self.call_reducer_inner( caller_identity, caller_connection_id, + tx_id, client, request_id, timer, @@ -1699,6 +1780,7 @@ impl ModuleHost { &self, caller_identity: Identity, caller_connection_id: Option, + tx_id: Option, client: Option>, request_id: Option, timer: Option, @@ -1722,6 +1804,7 @@ impl ModuleHost { self.call_reducer_inner_with_return( caller_identity, caller_connection_id, + tx_id, client, request_id, timer, @@ -1755,6 +1838,7 @@ impl ModuleHost { &self, caller_identity: Identity, caller_connection_id: Option, + tx_id: Option, reducer_name: &str, args: FunctionArgs, ) -> Result<(String, ReducerCallResult, Option), ReducerCallError> { @@ -1781,6 +1865,7 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + tx_id, client: None, request_id: None, timer: None, @@ -1790,10 +1875,12 @@ impl ModuleHost { // Include the coordinator identity so prepare_ids from different coordinators // cannot collide on the participant's st_2pc_state table. - let coordinator_hex = caller_identity.to_hex(); + let prepare_tx_component = tx_id + .map(|tx_id| tx_id.to_string()) + .unwrap_or_else(|| format!("legacy:{}:00000000", caller_identity.to_hex())); let prepare_id = format!( "prepare-{}-{}", - &coordinator_hex.to_string()[..16], + prepare_tx_component, PREPARE_COUNTER.fetch_add(1, Ordering::Relaxed), ); @@ -1808,6 +1895,39 @@ impl ModuleHost { decision_sender: decision_tx, }, ); + if let Some(tx_id) = tx_id { + let session = self + .replica_ctx() + .global_tx_manager + .ensure_session(tx_id, super::global_tx::GlobalTxRole::Participant, tx_id.creator_db); + session.set_state(super::global_tx::GlobalTxState::Preparing); + self.replica_ctx() + .global_tx_manager + .set_prepare_mapping(tx_id, prepare_id.clone()); + match self.acquire_global_tx_slot(tx_id).await { + Ok(guard) => { + guard.disarm(); + } + Err(outcome) => { + self.prepared_txs.remove(&prepare_id); + self.replica_ctx().global_tx_manager.remove_prepare_mapping(&prepare_id); + self.replica_ctx() + .global_tx_manager + .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborted); + self.replica_ctx().global_tx_manager.release(&tx_id); + self.replica_ctx().global_tx_manager.remove_session(&tx_id); + return Ok(( + String::new(), + ReducerCallResult { + outcome, + energy_used: EnergyQuanta::ZERO, + execution_duration: Default::default(), + }, + None, + )); + } + } + } // Spawn a background task that runs the reducer and holds the write lock // until we send a decision. The executor thread blocks inside @@ -1849,10 +1969,23 @@ impl ModuleHost { match prepared_rx.await { Ok((result, return_value)) => { if matches!(result.outcome, ReducerOutcome::Committed) { + if let Some(tx_id) = tx_id { + self.replica_ctx() + .global_tx_manager + .mark_state(&tx_id, super::global_tx::GlobalTxState::Prepared); + } Ok((prepare_id, result, return_value)) } else { // Reducer failed — remove the entry we registered (no hold in progress). self.prepared_txs.remove(&prepare_id); + if let Some(tx_id) = tx_id { + self.replica_ctx().global_tx_manager.remove_prepare_mapping(&prepare_id); + self.replica_ctx() + .global_tx_manager + .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborted); + self.replica_ctx().global_tx_manager.release(&tx_id); + self.replica_ctx().global_tx_manager.remove_session(&tx_id); + } Ok((String::new(), result, return_value)) } } @@ -1862,6 +1995,11 @@ impl ModuleHost { /// Finalize a prepared transaction as COMMIT. pub fn commit_prepared(&self, prepare_id: &str) -> Result<(), String> { + if let Some(tx_id) = self.replica_ctx().global_tx_manager.remove_prepare_mapping(prepare_id) { + self.replica_ctx() + .global_tx_manager + .mark_state(&tx_id, super::global_tx::GlobalTxState::Committing); + } let info = self .prepared_txs .remove(prepare_id) @@ -1873,6 +2011,11 @@ impl ModuleHost { /// Abort a prepared transaction. pub fn abort_prepared(&self, prepare_id: &str) -> Result<(), String> { + if let Some(tx_id) = self.replica_ctx().global_tx_manager.remove_prepare_mapping(prepare_id) { + self.replica_ctx() + .global_tx_manager + .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborting); + } let info = self .prepared_txs .remove(prepare_id) @@ -1882,6 +2025,175 @@ impl ModuleHost { Ok(()) } + pub async fn wound_global_tx(&self, tx_id: GlobalTxId) -> Result<(), String> { + let local_db = self.replica_ctx().database.database_identity; + if tx_id.creator_db != local_db { + return Err(format!( + "global transaction {tx_id} is coordinated by {}, not {}", + tx_id.creator_db, local_db + )); + } + + let Some(session) = self.replica_ctx().global_tx_manager.get_session(&tx_id) else { + return Ok(()); + }; + match session.state() { + super::global_tx::GlobalTxState::Committed + | super::global_tx::GlobalTxState::Aborted + | super::global_tx::GlobalTxState::Aborting => return Ok(()), + _ => {} + } + let session = self + .replica_ctx() + .global_tx_manager + .wound(&tx_id) + .ok_or_else(|| format!("no such global transaction: {tx_id}"))?; + + if let Some(prepare_id) = session.prepare_id() { + let _ = self.abort_prepared(&prepare_id); + } + + let participants = session.participants(); + let client = self.replica_ctx().call_reducer_client.clone(); + let router = self.replica_ctx().call_reducer_router.clone(); + let auth_token = self.replica_ctx().call_reducer_auth_token.clone(); + for (participant_identity, prepare_id) in participants { + if participant_identity == local_db { + let _ = self.abort_prepared(&prepare_id); + continue; + } + + let base_url = router + .resolve_base_url(participant_identity) + .await + .map_err(|e| format!("failed to resolve participant {participant_identity}: {e}"))?; + let url = format!( + "{}/v1/database/{}/2pc/abort/{}", + base_url, + participant_identity.to_hex(), + prepare_id, + ); + let mut req = client.post(&url); + if let Some(token) = &auth_token { + req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + } + match req.send().await { + Ok(resp) if resp.status().is_success() => {} + Ok(resp) => { + log::warn!( + "2PC wound: participant abort for {prepare_id} on {participant_identity} returned {}", + resp.status() + ); + } + Err(e) => { + log::warn!( + "2PC wound: transport error aborting {prepare_id} on {participant_identity}: {e}" + ); + } + } + } + + Ok(()) + } + + async fn acquire_global_tx_slot(&self, tx_id: GlobalTxId) -> Result, ReducerOutcome> { + let manager = &self.replica_ctx().global_tx_manager; + let local_db = self.replica_ctx().database.database_identity; + let role = if tx_id.creator_db == local_db { + super::global_tx::GlobalTxRole::Coordinator + } else { + super::global_tx::GlobalTxRole::Participant + }; + manager.ensure_session(tx_id, role, tx_id.creator_db); + if let Some(outcome) = self.check_global_tx_wounded(tx_id) { + self.abort_global_tx_locally(tx_id, true); + return Err(outcome); + } + + loop { + match manager.acquire(tx_id).await { + super::global_tx::AcquireDisposition::Acquired => { + if let Some(outcome) = self.check_global_tx_wounded(tx_id) { + self.abort_global_tx_locally(tx_id, true); + return Err(outcome); + } + return Ok(GlobalTxAdmissionGuard::new(self, tx_id)); + } + super::global_tx::AcquireDisposition::Wound(owner) => { + let _ = manager.wound(&owner); + if owner.creator_db != local_db { + self.send_wound_to_coordinator(owner).await; + } + } + super::global_tx::AcquireDisposition::Wait => {} + } + } + } + + fn check_global_tx_wounded(&self, tx_id: GlobalTxId) -> Option { + self.replica_ctx() + .global_tx_manager + .is_wounded(&tx_id) + .then(|| ReducerOutcome::Wounded(Box::new(Box::from(Self::wounded_message(tx_id))))) + } + + fn abort_global_tx_locally(&self, tx_id: GlobalTxId, remove_session: bool) { + self.replica_ctx() + .global_tx_manager + .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborted); + self.replica_ctx().global_tx_manager.release(&tx_id); + if remove_session { + self.replica_ctx().global_tx_manager.remove_session(&tx_id); + } + } + + fn wounded_message(tx_id: GlobalTxId) -> String { + format!("distributed transaction {tx_id} was wounded by an older transaction") + } + + fn tx_id_from_prepare_id(prepare_id: &str) -> Option { + let raw = prepare_id.strip_prefix("prepare-")?; + let (tx_component, _) = raw.rsplit_once('-')?; + if tx_component.starts_with("legacy:") { + return None; + } + tx_component.parse().ok() + } + + async fn send_wound_to_coordinator(&self, tx_id: GlobalTxId) { + let client = self.replica_ctx().call_reducer_client.clone(); + let router = self.replica_ctx().call_reducer_router.clone(); + let auth_token = self.replica_ctx().call_reducer_auth_token.clone(); + let base_url = match router.resolve_base_url(tx_id.creator_db).await { + Ok(url) => url, + Err(e) => { + log::warn!("2PC wound: cannot resolve coordinator URL for {tx_id}: {e}"); + return; + } + }; + let url = format!( + "{}/v1/database/{}/2pc/wound/{}", + base_url, + tx_id.creator_db.to_hex(), + tx_id, + ); + let mut req = client.post(&url); + if let Some(token) = &auth_token { + req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + } + match req.send().await { + Ok(resp) if resp.status().is_success() => { + log::info!("2PC wound: notified coordinator for {tx_id}"); + } + Ok(resp) => { + log::warn!("2PC wound: coordinator returned {} for {tx_id}", resp.status()); + } + Err(e) => { + log::warn!("2PC wound: transport error for {tx_id}: {e}"); + } + } + } + /// Delete a coordinator log entry for `prepare_id`. /// Called when B has confirmed it committed, so A can stop retransmitting. pub fn ack_2pc_coordinator_commit(&self, prepare_id: &str) -> Result<(), anyhow::Error> { @@ -1925,6 +2237,7 @@ impl ModuleHost { let auth_token = replica_ctx.call_reducer_auth_token.clone(); for row in rows { let prepare_id = row.participant_prepare_id.clone(); + let recovered_tx_id = Self::tx_id_from_prepare_id(&prepare_id); let participant_identity = match Identity::from_hex(&row.participant_identity_hex) { Ok(id) => id, Err(e) => { @@ -1935,6 +2248,15 @@ impl ModuleHost { continue; } }; + if let Some(tx_id) = recovered_tx_id { + let session = replica_ctx.global_tx_manager.ensure_session( + tx_id, + super::global_tx::GlobalTxRole::Coordinator, + tx_id.creator_db, + ); + session.set_state(super::global_tx::GlobalTxState::Committing); + session.add_participant(participant_identity, prepare_id.clone()); + } let base_url = match router.resolve_base_url(participant_identity).await { Ok(url) => url, Err(e) => { @@ -1960,6 +2282,13 @@ impl ModuleHost { }) { log::warn!("recover_2pc_coordinator: delete coordinator log failed for {prepare_id}: {e}"); } + if let Some(tx_id) = recovered_tx_id { + replica_ctx + .global_tx_manager + .mark_state(&tx_id, super::global_tx::GlobalTxState::Committed); + replica_ctx.global_tx_manager.release(&tx_id); + replica_ctx.global_tx_manager.remove_session(&tx_id); + } } Ok(resp) => { log::warn!( @@ -2019,13 +2348,34 @@ impl ModuleHost { .map(ConnectionId::from_u128) .unwrap_or(ConnectionId::ZERO); let args = FunctionArgs::Bsatn(row.args_bsatn.clone().into()); + let recovered_tx_id = Self::tx_id_from_prepare_id(&original_prepare_id); + if let Some(tx_id) = recovered_tx_id { + let session = this.replica_ctx().global_tx_manager.ensure_session( + tx_id, + super::global_tx::GlobalTxRole::Participant, + coordinator_identity, + ); + session.set_state(super::global_tx::GlobalTxState::Prepared); + this.replica_ctx() + .global_tx_manager + .set_prepare_mapping(tx_id, original_prepare_id.clone()); + } // Step 1: Re-run the reducer to reacquire the write lock. let new_prepare_id = match this - .prepare_reducer(caller_identity, Some(caller_connection_id), &row.reducer_name, args) + .prepare_reducer( + caller_identity, + Some(caller_connection_id), + recovered_tx_id, + &row.reducer_name, + args, + ) .await { Ok((pid, result, _rv)) if !pid.is_empty() => { + this.replica_ctx() + .global_tx_manager + .remove_prepare_mapping(&original_prepare_id); log::info!( "recover_2pc_participant: re-prepared {original_prepare_id} as {pid}: {:?}", result.outcome diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 5810c67d7bd..265f271d59e 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -327,8 +327,14 @@ impl JsInstanceEnv { /// /// Returns the handle used by reducers to read from `args` /// as well as the handle used to write the error message, if any. - fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType) { - self.instance_env.start_funcall(name, ts, func_type); + fn start_funcall( + &mut self, + name: Identifier, + ts: Timestamp, + func_type: FuncCallType, + tx_id: Option, + ) { + self.instance_env.start_funcall(name, ts, func_type, tx_id); } /// Returns the name of the most recent reducer to be run in this environment, @@ -613,7 +619,7 @@ enum JsWorkerRequest { }, } -static_assert_size!(CallReducerParams, 192); +static_assert_size!(CallReducerParams, 256); fn send_worker_reply(ctx: &str, reply_tx: JsReplyTx, value: T, trapped: bool) { if reply_tx.send(JsWorkerReply { value, trapped }).is_err() { @@ -1440,7 +1446,12 @@ where // Start the timer. // We'd like this tightly around `call`. - env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); + env.start_funcall( + op.name().clone(), + op.timestamp(), + op.call_type(), + op.tx_id(), + ); v8::tc_scope!(scope, scope); let call_result = call(scope, op).map_err(|mut e| { @@ -1543,6 +1554,7 @@ mod test { caller_identity: &Identity::ONE, caller_connection_id: &ConnectionId::ZERO, timestamp: Timestamp::from_micros_since_unix_epoch(24), + tx_id: None, args: &ArgsTuple::nullary(), }; let buffer = v8::ArrayBuffer::new(scope, 4096); diff --git a/crates/core/src/host/v8/syscall/v1.rs b/crates/core/src/host/v8/syscall/v1.rs index 2ccd36620dd..2884ab8463e 100644 --- a/crates/core/src/host/v8/syscall/v1.rs +++ b/crates/core/src/host/v8/syscall/v1.rs @@ -496,6 +496,7 @@ pub(super) fn call_call_reducer( caller_identity: sender, caller_connection_id: conn_id, timestamp, + tx_id: _, args: reducer_args, } = op; // Serialize the arguments. diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index 5f3f6ed3edb..ce4d0ddfff1 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -427,6 +427,7 @@ pub(super) fn call_call_reducer<'scope>( caller_identity: sender, caller_connection_id: conn_id, timestamp, + tx_id: _, args: reducer_args, } = op; // Serialize the arguments. diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 5d744bc2108..30e6689e532 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -357,6 +357,7 @@ pub fn err_to_errno(err: NodesError) -> Result<(NonZeroU16, Option), Nod NodesError::IndexRowNotFound => errno::NO_SUCH_ROW, NodesError::IndexCannotSeekRange => errno::WRONG_INDEX_ALGO, NodesError::ScheduleError(ScheduleError::DelayTooLong(_)) => errno::SCHEDULE_AT_DELAY_TOO_LONG, + NodesError::Wounded(message) => return Ok((errno::HTTP_ERROR, Some(message))), NodesError::HttpError(message) => return Ok((errno::HTTP_ERROR, Some(message))), NodesError::Internal(ref internal) => match **internal { DBError::Datastore(DatastoreError::Index(IndexError::UniqueConstraintViolation( diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 1301eaf447a..faa21a48794 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -444,6 +444,18 @@ async fn send_ack_commit_to_coordinator( } } +fn wounded_status(replica_ctx: &ReplicaContext, tx_id: spacetimedb_lib::GlobalTxId) -> EventStatus { + let _ = replica_ctx.global_tx_manager.wound(&tx_id); + EventStatus::Wounded(format!( + "distributed transaction {tx_id} was wounded by an older transaction" + )) +} + +fn check_wounded(replica_ctx: &ReplicaContext, tx_id: Option) -> Option { + tx_id.filter(|tx_id| replica_ctx.global_tx_manager.is_wounded(tx_id)) + .map(|tx_id| wounded_status(replica_ctx, tx_id)) +} + impl WasmModuleHostActor { fn make_from_instance(&self, mut instance: T::Instance) -> WasmModuleInstance { let common = InstanceCommon::new(&self.common); @@ -637,6 +649,7 @@ impl WasmModuleInstance { ) { let stdb = self.instance.replica_ctx().relational_db().clone(); let replica_ctx = self.instance.replica_ctx().clone(); + let global_tx_id = params.tx_id; // Extract recovery info before params are consumed. let recovery_reducer_name = self @@ -652,7 +665,7 @@ impl WasmModuleInstance { let recovery_timestamp_micros = params.timestamp.to_micros_since_unix_epoch(); // Step 1: run the reducer and hold the write lock open. - let (mut tx, event, client, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { + let (mut tx, mut event, client, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { self.common.run_reducer_no_commit(None, params, &mut self.instance) }); self.trapped = trapped; @@ -660,6 +673,13 @@ impl WasmModuleInstance { let energy_quanta_used = event.energy_quanta_used; let total_duration = event.host_execution_duration; + if let Some(status) = check_wounded(&replica_ctx, global_tx_id) + && matches!(event.status, EventStatus::Committed(_)) + { + event.status = status; + event.reducer_return_value = None; + } + if !matches!(event.status, EventStatus::Committed(_)) { // Reducer failed — roll back and signal failure; no marker was written. let res = ReducerCallResult { @@ -670,6 +690,13 @@ impl WasmModuleInstance { let return_value = event.reducer_return_value.clone(); let _ = prepared_tx.send((res, return_value)); let _ = stdb.rollback_mut_tx(tx); + if let Some(tx_id) = global_tx_id { + replica_ctx + .global_tx_manager + .mark_state(&tx_id, crate::host::global_tx::GlobalTxState::Aborted); + replica_ctx.global_tx_manager.release(&tx_id); + replica_ctx.global_tx_manager.remove_session(&tx_id); + } return; } @@ -713,7 +740,10 @@ impl WasmModuleInstance { let _ = prepared_tx.send((res, return_value)); // Step 4: wait for coordinator's decision (B never aborts on its own). - let commit = Self::wait_for_2pc_decision(decision_rx, &prepare_id, coordinator_identity, &replica_ctx); + let commit = !global_tx_id + .map(|tx_id| replica_ctx.global_tx_manager.is_wounded(&tx_id)) + .unwrap_or(false) + && Self::wait_for_2pc_decision(decision_rx, &prepare_id, coordinator_identity, &replica_ctx); if commit { // Delete the marker in the same tx as the reducer changes (atomic commit). @@ -748,6 +778,13 @@ impl WasmModuleInstance { coordinator_identity, prepare_id, )); + if let Some(tx_id) = global_tx_id { + replica_ctx + .global_tx_manager + .mark_state(&tx_id, crate::host::global_tx::GlobalTxState::Committed); + replica_ctx.global_tx_manager.release(&tx_id); + replica_ctx.global_tx_manager.remove_session(&tx_id); + } } else { // ABORT: roll back reducer changes; clean up the already-committed marker. let _ = stdb.rollback_mut_tx(tx); @@ -758,6 +795,13 @@ impl WasmModuleInstance { "call_reducer_prepare_and_hold: abort: failed to delete st_2pc_state for {prepare_id}: {e}" ); } + if let Some(tx_id) = global_tx_id { + replica_ctx + .global_tx_manager + .mark_state(&tx_id, crate::host::global_tx::GlobalTxState::Aborted); + replica_ctx.global_tx_manager.release(&tx_id); + replica_ctx.global_tx_manager.remove_session(&tx_id); + } } } @@ -1123,7 +1167,8 @@ impl InstanceCommon { params: CallReducerParams, inst: &mut I, ) -> (ReducerCallResult, Option, bool) { - let (mut tx, event, client, trapped) = self.run_reducer_no_commit(tx, params, inst); + let managed_global_tx_id = if tx.is_none() { params.tx_id } else { None }; + let (mut tx, mut event, client, trapped) = self.run_reducer_no_commit(tx, params, inst); let energy_quanta_used = event.energy_quanta_used; let total_duration = event.host_execution_duration; @@ -1143,6 +1188,13 @@ impl InstanceCommon { } } + if let Some(status) = check_wounded(inst.replica_ctx(), managed_global_tx_id) + && matches!(event.status, EventStatus::Committed(_)) + { + event.status = status; + event.reducer_return_value = None; + } + let commit_result = commit_and_broadcast_event(&self.info.subscriptions, client, event, tx); let commit_tx_offset = commit_result.tx_offset; let event = commit_result.event; @@ -1216,6 +1268,20 @@ impl InstanceCommon { }); } + if let Some(tx_id) = managed_global_tx_id { + let manager = &inst.replica_ctx().global_tx_manager; + manager.mark_state( + &tx_id, + if matches!(event.status, EventStatus::Committed(_)) { + crate::host::global_tx::GlobalTxState::Committed + } else { + crate::host::global_tx::GlobalTxState::Aborted + }, + ); + manager.release(&tx_id); + manager.remove_session(&tx_id); + } + let res = ReducerCallResult { outcome: ReducerOutcome::from(&event.status), energy_used: energy_quanta_used, @@ -1248,6 +1314,7 @@ impl InstanceCommon { timestamp, caller_identity, caller_connection_id, + tx_id, client, request_id, reducer_id, @@ -1256,7 +1323,7 @@ impl InstanceCommon { } = params; let caller_connection_id_opt = (caller_connection_id != ConnectionId::ZERO).then_some(caller_connection_id); - let replica_ctx = inst.replica_ctx(); + let replica_ctx = inst.replica_ctx().clone(); let stdb = replica_ctx.relational_db(); let info = self.info.clone(); let reducer_def = info.module_def.reducer_by_id(reducer_id); @@ -1272,6 +1339,7 @@ impl InstanceCommon { caller_identity: &caller_identity, caller_connection_id: &caller_connection_id, timestamp, + tx_id, args: &args, }; @@ -1341,6 +1409,15 @@ impl InstanceCommon { } }; + let status = if let Some(status) = check_wounded(&replica_ctx, tx_id) + && matches!(status, EventStatus::Committed(_)) + { + reducer_return_value = None; + status + } else { + status + }; + // Only re-evaluate and update views if the reducer's execution was successful let (out, trapped) = if !trapped && matches!(status, EventStatus::Committed(_)) { self.call_views_with_tx(tx, caller_identity, inst, timestamp) @@ -1353,11 +1430,16 @@ impl InstanceCommon { vm_metrics.report_total_duration(out.total_duration); vm_metrics.report_abi_duration(out.abi_duration); - let status = match &out.outcome { + let mut status = match &out.outcome { ViewOutcome::BudgetExceeded => EventStatus::OutOfEnergy, ViewOutcome::Failed(err) => EventStatus::FailedInternal(err.clone()), ViewOutcome::Success => status, }; + if let Some(wounded) = check_wounded(&replica_ctx, tx_id) + && matches!(status, EventStatus::Committed(_)) + { + status = wounded; + } if !matches!(status, EventStatus::Committed(_)) { reducer_return_value = None; } @@ -2032,6 +2114,9 @@ pub trait InstanceOp { fn name(&self) -> &Identifier; fn timestamp(&self) -> Timestamp; fn call_type(&self) -> FuncCallType; + fn tx_id(&self) -> Option { + None + } } /// Describes a view call in a cheaply shareable way. @@ -2103,6 +2188,7 @@ pub struct ReducerOp<'a> { pub caller_identity: &'a Identity, pub caller_connection_id: &'a ConnectionId, pub timestamp: Timestamp, + pub tx_id: Option, /// The arguments passed to the reducer. pub args: &'a ArgsTuple, } @@ -2117,6 +2203,9 @@ impl InstanceOp for ReducerOp<'_> { fn call_type(&self) -> FuncCallType { FuncCallType::Reducer } + fn tx_id(&self) -> Option { + self.tx_id + } } impl From> for execution_context::ReducerContext { @@ -2127,6 +2216,7 @@ impl From> for execution_context::ReducerContext { caller_identity, caller_connection_id, timestamp, + tx_id: _, args, }: ReducerOp<'_>, ) -> Self { diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index e23e3da01bb..6f59a46471f 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -17,7 +17,7 @@ use crate::subscription::module_subscription_manager::TransactionOffset; use anyhow::{anyhow, Context as _}; use spacetimedb_data_structures::map::IntMap; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; -use spacetimedb_lib::{bsatn, ConnectionId, Identity, Timestamp}; +use spacetimedb_lib::{bsatn, ConnectionId, GlobalTxId, Identity, Timestamp}; use spacetimedb_primitives::errno::HOST_CALL_FAILURE; use spacetimedb_primitives::{errno, ColId}; use spacetimedb_schema::def::ModuleDef; @@ -259,6 +259,7 @@ impl WasmInstanceEnv { args: bytes::Bytes, ts: Timestamp, func_type: FuncCallType, + tx_id: Option, ) -> (BytesSourceId, u32) { // Create the output sink. // Reducers which fail will write their error message here. @@ -267,7 +268,7 @@ impl WasmInstanceEnv { let args = self.create_bytes_source(args).unwrap(); - self.instance_env.start_funcall(name, ts, func_type); + self.instance_env.start_funcall(name, ts, func_type, tx_id); (args, errors) } @@ -2008,7 +2009,7 @@ impl WasmInstanceEnv { bytes_source.0.write_to(mem, out)?; Ok(status as u32) } - Err(NodesError::HttpError(err)) => { + Err(NodesError::HttpError(err) | NodesError::Wounded(err)) => { let err_bytes = bsatn::to_vec(&err).with_context(|| { format!("Failed to BSATN-serialize call_reducer_on_db transport error: {err:?}") })?; @@ -2067,13 +2068,21 @@ impl WasmInstanceEnv { if let Some(pid) = prepare_id && status < 300 { + if let Some(tx_id) = env.instance_env.current_tx_id() { + let session = env.instance_env.replica_ctx.global_tx_manager.ensure_session( + tx_id, + crate::host::global_tx::GlobalTxRole::Coordinator, + tx_id.creator_db, + ); + session.add_participant(database_identity, pid.clone()); + } env.instance_env.prepared_participants.push((database_identity, pid)); } let bytes_source = WasmInstanceEnv::create_bytes_source(env, body)?; bytes_source.0.write_to(mem, out)?; Ok(status as u32) } - Err(NodesError::HttpError(err)) => { + Err(NodesError::HttpError(err) | NodesError::Wounded(err)) => { let err_bytes = bsatn::to_vec(&err).with_context(|| { format!("Failed to BSATN-serialize call_reducer_on_db_2pc transport error: {err:?}") })?; diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index fb01e3a4763..878a0be8b96 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -463,7 +463,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (args_source, errors_sink) = store .data_mut() - .start_funcall(reducer_name, args_bytes, op.timestamp, op.call_type()); + .start_funcall(reducer_name, args_bytes, op.timestamp, op.call_type(), op.tx_id); let call_result = call_sync_typed_func( &self.call_reducer, @@ -502,7 +502,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (args_source, errors_sink) = store .data_mut() - .start_funcall(op.name.clone(), args_bytes, op.timestamp, op.call_type()); + .start_funcall(op.name.clone(), args_bytes, op.timestamp, op.call_type(), None); let call_result = call_view_export( &mut *store, @@ -538,7 +538,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (args_source, errors_sink) = store .data_mut() - .start_funcall(op.name.clone(), args_bytes, op.timestamp, op.call_type()); + .start_funcall(op.name.clone(), args_bytes, op.timestamp, op.call_type(), None); let call_result = call_view_export( &mut *store, @@ -585,7 +585,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (args_source, result_sink) = store .data_mut() - .start_funcall(op.name.clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); + .start_funcall(op.name.clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure, None); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { diff --git a/crates/core/src/replica_context.rs b/crates/core/src/replica_context.rs index 8c9f8804f24..aa190f13225 100644 --- a/crates/core/src/replica_context.rs +++ b/crates/core/src/replica_context.rs @@ -3,11 +3,13 @@ use spacetimedb_commitlog::SizeOnDisk; use super::database_logger::DatabaseLogger; use crate::db::relational_db::RelationalDB; use crate::error::DBError; +use crate::host::global_tx::GlobalTxManager; use crate::host::reducer_router::ReducerCallRouter; use crate::messages::control_db::Database; use crate::subscription::module_subscription_actor::ModuleSubscriptions; use std::io; use std::ops::Deref; +use std::sync::atomic::AtomicU32; use std::sync::Arc; use std::time::Duration; @@ -64,6 +66,10 @@ pub struct ReplicaContext { /// /// `None` in contexts where no auth token is configured (e.g. unit tests). pub call_reducer_auth_token: Option, + /// Per-database nonce used when minting reducer transaction ids. + pub tx_id_nonce: Arc, + /// In-memory distributed transaction sessions and lock scheduling state. + pub global_tx_manager: Arc, } impl ReplicaContext { diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index 92f296f3b8c..635808dd48f 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -346,6 +346,7 @@ impl ModuleSubscriptions { let error = match &event.status { EventStatus::FailedUser(err) => err.clone(), EventStatus::FailedInternal(err) => err.clone(), + EventStatus::Wounded(err) => err.clone(), EventStatus::OutOfEnergy => "reducer ran out of energy".into(), EventStatus::Committed(_) => { tracing::warn!("Unexpected committed status in reducer failure branch"); @@ -1616,7 +1617,10 @@ impl ModuleSubscriptions { *db_update = DatabaseUpdate::from_writes(&tx_data); (read_tx, tx_data, tx_metrics) } - EventStatus::FailedUser(_) | EventStatus::FailedInternal(_) | EventStatus::OutOfEnergy => { + EventStatus::FailedUser(_) + | EventStatus::FailedInternal(_) + | EventStatus::Wounded(_) + | EventStatus::OutOfEnergy => { // If the transaction failed, we need to rollback the mutable tx. // We don't need to do any subscription updates in this case, so we will exit early. diff --git a/crates/datastore/src/execution_context.rs b/crates/datastore/src/execution_context.rs index f2e24a5876e..d740c09aba3 100644 --- a/crates/datastore/src/execution_context.rs +++ b/crates/datastore/src/execution_context.rs @@ -79,7 +79,6 @@ impl TryFrom<&txdata::Inputs> for ReducerContext { let caller_identity = bsatn::from_reader(args)?; let caller_connection_id = bsatn::from_reader(args)?; let timestamp = bsatn::from_reader(args)?; - let name = RawIdentifier::new(&**inputs.reducer_name); let name = ReducerName::new(Identifier::new_assume_valid(name)); diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 2e8b9c08336..7bbeb3d57fc 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -22,6 +22,7 @@ pub mod operator; pub mod query; pub mod scheduler; pub mod st_var; +pub mod tx_id; pub mod version; pub mod type_def { @@ -38,6 +39,7 @@ pub use filterable_value::Private; pub use filterable_value::{FilterableValue, IndexScanRangeBoundsTerminator, TermBound}; pub use identity::Identity; pub use scheduler::ScheduleAt; +pub use tx_id::{GlobalTxId, TX_ID_HEADER}; pub use spacetimedb_sats::hash::{self, hash_bytes, Hash}; pub use spacetimedb_sats::time_duration::TimeDuration; pub use spacetimedb_sats::timestamp::Timestamp; diff --git a/crates/lib/src/tx_id.rs b/crates/lib/src/tx_id.rs new file mode 100644 index 00000000000..8e67829fbb9 --- /dev/null +++ b/crates/lib/src/tx_id.rs @@ -0,0 +1,64 @@ +use crate::{Identity, SpacetimeType, Timestamp}; +use std::fmt; +use std::str::FromStr; + +/// Header used to propagate distributed reducer transaction ids across remote calls. +pub const TX_ID_HEADER: &str = "X-Spacetime-Tx-Id"; + +/// A distributed reducer transaction identifier. +/// +/// Ordering is primarily by `start_ts`, so this can later support wound-wait. +/// `creator_db` namespaces the id globally, and `nonce` breaks ties for +/// multiple transactions started on the same database at the same timestamp. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, SpacetimeType)] +#[sats(crate = crate)] +pub struct GlobalTxId { + pub start_ts: Timestamp, + pub creator_db: Identity, + pub nonce: u32, +} + +impl GlobalTxId { + pub const fn new(start_ts: Timestamp, creator_db: Identity, nonce: u32) -> Self { + Self { + start_ts, + creator_db, + nonce, + } + } +} + +impl fmt::Display for GlobalTxId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}:{}:{:08x}", + self.start_ts.to_micros_since_unix_epoch(), + self.creator_db.to_hex(), + self.nonce + ) + } +} + +impl FromStr for GlobalTxId { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let mut parts = s.splitn(3, ':'); + let start_ts = parts.next().ok_or("missing tx timestamp")?; + let creator_db = parts.next().ok_or("missing tx creator db")?; + let nonce = parts.next().ok_or("missing tx nonce")?; + if parts.next().is_some() { + return Err("too many tx id components"); + } + + let start_ts = start_ts + .parse::() + .map(Timestamp::from_micros_since_unix_epoch) + .map_err(|_| "invalid tx timestamp")?; + let creator_db = Identity::from_hex(creator_db).map_err(|_| "invalid tx creator db")?; + let nonce = u32::from_str_radix(nonce, 16).map_err(|_| "invalid tx nonce")?; + + Ok(Self::new(start_ts, creator_db, nonce)) + } +} From b9bc1537893e84be97fda87a8a4a42d3cc85c6d2 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Mon, 30 Mar 2026 09:22:53 -0700 Subject: [PATCH 02/33] WIP fixing lock issues --- crates/core/src/host/global_tx.rs | 214 +++++++++++++++++++++++---- crates/core/src/host/instance_env.rs | 1 + crates/core/src/host/module_host.rs | 44 +++--- 3 files changed, 210 insertions(+), 49 deletions(-) diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index bf478ef7c80..d593795d3ae 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -1,9 +1,10 @@ use crate::identity::Identity; use spacetimedb_lib::GlobalTxId; use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use tokio::sync::Notify; +use tokio::sync::{watch, Notify}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GlobalTxRole { @@ -28,6 +29,7 @@ pub struct GlobalTxSession { pub role: GlobalTxRole, pub coordinator_identity: Identity, wounded: AtomicBool, + wounded_tx: watch::Sender, state: Mutex, prepare_id: Mutex>, participants: Mutex>, @@ -35,11 +37,13 @@ pub struct GlobalTxSession { impl GlobalTxSession { fn new(tx_id: GlobalTxId, role: GlobalTxRole, coordinator_identity: Identity) -> Self { + let (wounded_tx, _) = watch::channel(false); Self { tx_id, role, coordinator_identity, wounded: AtomicBool::new(false), + wounded_tx, state: Mutex::new(GlobalTxState::Running), prepare_id: Mutex::new(None), participants: Mutex::new(HashMap::new()), @@ -51,7 +55,15 @@ impl GlobalTxSession { } pub fn wound(&self) -> bool { - !self.wounded.swap(true, Ordering::SeqCst) + let was_fresh = !self.wounded.swap(true, Ordering::SeqCst); + if was_fresh { + let _ = self.wounded_tx.send(true); + } + was_fresh + } + + pub fn subscribe_wounded(&self) -> watch::Receiver { + self.wounded_tx.subscribe() } pub fn state(&self) -> GlobalTxState { @@ -100,11 +112,39 @@ impl Default for LockState { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AcquireDisposition { - Acquired, - Wound(GlobalTxId), - Wait, +pub enum AcquireDisposition<'a> { + Acquired(GlobalTxLockGuard<'a>), + Cancelled, +} + +pub struct GlobalTxLockGuard<'a> { + manager: &'a GlobalTxManager, + tx_id: Option, +} + +impl<'a> GlobalTxLockGuard<'a> { + fn new(manager: &'a GlobalTxManager, tx_id: GlobalTxId) -> Self { + Self { + manager, + tx_id: Some(tx_id), + } + } + + pub fn tx_id(&self) -> GlobalTxId { + self.tx_id.expect("lock guard must always have a tx_id before drop") + } + + pub fn disarm(mut self) { + self.tx_id = None; + } +} + +impl Drop for GlobalTxLockGuard<'_> { + fn drop(&mut self) { + if let Some(tx_id) = self.tx_id.take() { + self.manager.release(&tx_id); + } + } } #[derive(Default)] @@ -174,6 +214,10 @@ impl GlobalTxManager { self.get_session(tx_id).map(|s| s.is_wounded()).unwrap_or(false) } + pub fn subscribe_wounded(&self, tx_id: &GlobalTxId) -> Option> { + self.get_session(tx_id).map(|s| s.subscribe_wounded()) + } + pub fn wound(&self, tx_id: &GlobalTxId) -> Option> { let session = self.get_session(tx_id)?; let _ = session.wound(); @@ -183,30 +227,56 @@ impl GlobalTxManager { Some(session) } - pub async fn acquire(&self, tx_id: GlobalTxId) -> AcquireDisposition { + pub async fn acquire(&self, tx_id: GlobalTxId, mut on_wound: F) -> AcquireDisposition<'_> + where + F: FnMut(GlobalTxId) -> Fut, + Fut: Future, + { + let mut wounded_rx = match self.subscribe_wounded(&tx_id) { + Some(rx) => rx, + None => return AcquireDisposition::Cancelled, + }; loop { - let waiter = { + // self.is_wounded(&tx_id) + if *wounded_rx.borrow() { + self.remove_waiter(&tx_id); + return AcquireDisposition::Cancelled; + } + + let (waiter, owner_to_wound) = { let mut state = self.lock_state.lock().unwrap(); match state.owner { None => { state.owner = Some(tx_id); state.waiting.remove(&tx_id); - return AcquireDisposition::Acquired; + return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } Some(owner) if owner == tx_id => { state.waiting.remove(&tx_id); - return AcquireDisposition::Acquired; + return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } Some(owner) => { state.waiting.insert(tx_id); - if tx_id < owner && state.wounded_owners.insert(owner) { - return AcquireDisposition::Wound(owner); - } - self.lock_notify.notified() + let owner_to_wound = (tx_id < owner && state.wounded_owners.insert(owner)).then_some(owner); + (self.lock_notify.notified(), owner_to_wound) } } }; - waiter.await; + + if let Some(owner) = owner_to_wound { + let _ = self.wound(&owner); + on_wound(owner).await; + } + + tokio::select! { + changed = wounded_rx.changed(), if !*wounded_rx.borrow() => { + if changed.is_ok() && *wounded_rx.borrow() { + self.remove_waiter(&tx_id); + return AcquireDisposition::Cancelled; + } + } + _ = waiter => {} + } } } @@ -219,6 +289,10 @@ impl GlobalTxManager { } state.waiting.remove(tx_id); } + + fn remove_waiter(&self, tx_id: &GlobalTxId) { + self.lock_state.lock().unwrap().waiting.remove(tx_id); + } } #[cfg(test)] @@ -227,8 +301,8 @@ mod tests { use crate::identity::Identity; use spacetimedb_lib::{GlobalTxId, Timestamp}; use std::sync::Arc; - use tokio::runtime::Runtime; use std::time::Duration; + use tokio::runtime::Runtime; fn tx_id(ts: i64, db_byte: u8, nonce: u32) -> GlobalTxId { GlobalTxId::new( @@ -240,7 +314,7 @@ mod tests { #[test] fn older_requester_wounds_younger_owner() { - let manager = GlobalTxManager::default(); + let manager = Arc::new(GlobalTxManager::default()); let younger = tx_id(20, 2, 0); let older = tx_id(10, 1, 0); manager.ensure_session( @@ -248,12 +322,28 @@ mod tests { super::GlobalTxRole::Participant, younger.creator_db, ); + manager.ensure_session(older, super::GlobalTxRole::Participant, older.creator_db); let rt = Runtime::new().unwrap(); - assert_eq!(rt.block_on(manager.acquire(younger)), AcquireDisposition::Acquired); - assert_eq!(rt.block_on(manager.acquire(older)), AcquireDisposition::Wound(younger)); - assert!(manager.wound(&younger).is_some()); + let younger_guard = match rt.block_on(manager.acquire(younger, |_| async {})) { + AcquireDisposition::Acquired(guard) => guard, + AcquireDisposition::Cancelled => panic!("younger tx should acquire immediately"), + }; + + let manager_for_task = manager.clone(); + let older_task = rt.spawn(async move { + match manager_for_task.acquire(older, |_| async {}).await { + AcquireDisposition::Acquired(_guard) => true, + AcquireDisposition::Cancelled => false, + } + }); + std::thread::sleep(Duration::from_millis(10)); assert!(manager.is_wounded(&younger)); + drop(younger_guard); + assert!(matches!( + rt.block_on(older_task).expect("task should complete"), + true + )); } #[test] @@ -261,13 +351,19 @@ mod tests { let manager = GlobalTxManager::default(); let older = tx_id(10, 1, 0); let younger = tx_id(20, 2, 0); + manager.ensure_session(older, super::GlobalTxRole::Participant, older.creator_db); + manager.ensure_session(younger, super::GlobalTxRole::Participant, younger.creator_db); let rt = Runtime::new().unwrap(); - assert_eq!(rt.block_on(manager.acquire(older)), AcquireDisposition::Acquired); + let older_guard = match rt.block_on(manager.acquire(older, |_| async {})) { + AcquireDisposition::Acquired(guard) => guard, + AcquireDisposition::Cancelled => panic!("older tx should acquire immediately"), + }; let wait = rt.block_on(async { - tokio::time::timeout(Duration::from_millis(25), manager.acquire(younger)).await + tokio::time::timeout(Duration::from_millis(25), manager.acquire(younger, |_| async {})).await }); assert!(wait.is_err()); + drop(older_guard); } #[test] @@ -275,19 +371,26 @@ mod tests { let manager = Arc::new(GlobalTxManager::default()); let owner = tx_id(10, 1, 0); let waiter = tx_id(20, 2, 0); + manager.ensure_session(owner, super::GlobalTxRole::Participant, owner.creator_db); + manager.ensure_session(waiter, super::GlobalTxRole::Participant, waiter.creator_db); let rt = Runtime::new().unwrap(); - assert_eq!(rt.block_on(manager.acquire(owner)), AcquireDisposition::Acquired); + let owner_guard = match rt.block_on(manager.acquire(owner, |_| async {})) { + AcquireDisposition::Acquired(guard) => guard, + AcquireDisposition::Cancelled => panic!("owner should acquire immediately"), + }; let manager_for_thread = manager.clone(); let handle = std::thread::spawn(move || { let rt = Runtime::new().unwrap(); - assert_eq!(rt.block_on(manager_for_thread.acquire(waiter)), AcquireDisposition::Acquired); - manager_for_thread.release(&waiter); + match rt.block_on(manager_for_thread.acquire(waiter, |_| async {})) { + AcquireDisposition::Acquired(_guard) => {} + AcquireDisposition::Cancelled => panic!("waiter should acquire after release"), + } }); std::thread::sleep(Duration::from_millis(25)); - manager.release(&owner); + drop(owner_guard); handle.join().unwrap(); } @@ -303,4 +406,61 @@ mod tests { assert!(manager.wound(&tx_id).is_some()); assert!(session.is_wounded()); } + + #[test] + fn wound_subscription_notifies_waiter() { + let manager = GlobalTxManager::default(); + let tx_id = tx_id(10, 1, 0); + let _session = manager.ensure_session(tx_id, super::GlobalTxRole::Coordinator, tx_id.creator_db); + let mut wounded_rx = manager.subscribe_wounded(&tx_id).expect("session should exist"); + + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let notifier = async { + if !*wounded_rx.borrow() { + wounded_rx.changed().await.expect("sender should still exist"); + } + *wounded_rx.borrow() + }; + + let trigger = async { + tokio::time::sleep(Duration::from_millis(10)).await; + manager.wound(&tx_id).expect("session should still exist"); + }; + + let (wounded, ()) = tokio::join!(notifier, trigger); + assert!(wounded); + }); + } + + #[test] + fn wounded_waiter_is_cancelled() { + let manager = Arc::new(GlobalTxManager::default()); + let owner = tx_id(10, 1, 0); + let waiter = tx_id(20, 2, 0); + manager.ensure_session(owner, super::GlobalTxRole::Participant, owner.creator_db); + manager.ensure_session(waiter, super::GlobalTxRole::Participant, waiter.creator_db); + + let rt = Runtime::new().unwrap(); + let owner_guard = match rt.block_on(manager.acquire(owner, |_| async {})) { + AcquireDisposition::Acquired(guard) => guard, + AcquireDisposition::Cancelled => panic!("owner should acquire immediately"), + }; + + let manager_for_task = manager.clone(); + let waiter_task = rt.spawn(async move { + matches!( + manager_for_task.acquire(waiter, |_| async {}).await, + AcquireDisposition::Cancelled + ) + }); + std::thread::sleep(Duration::from_millis(10)); + manager.wound(&waiter).expect("waiter session should exist"); + drop(owner_guard); + + assert!(matches!( + rt.block_on(waiter_task).expect("task should complete"), + true + )); + } } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 75eb421a354..cb787b7b66d 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1159,6 +1159,7 @@ impl InstanceEnv { if let Some(tx_id) = tx_id { req = req.header(TX_ID_HEADER, tx_id.to_string()); } + // TODO: This needs to select on subscribe_wounded as well, so we can stop waiting for the response when wounded. let result = async { let response = req.send().await.map_err(|e| NodesError::HttpError(e.to_string()))?; let status = response.status().as_u16(); diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index b984e695e37..0bf87089d35 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -233,27 +233,19 @@ pub struct ModuleInfo { } struct GlobalTxAdmissionGuard<'a> { - module_host: &'a ModuleHost, - tx_id: Option, + lock_guard: Option>, } impl<'a> GlobalTxAdmissionGuard<'a> { - fn new(module_host: &'a ModuleHost, tx_id: GlobalTxId) -> Self { + fn new(lock_guard: super::global_tx::GlobalTxLockGuard<'a>) -> Self { Self { - module_host, - tx_id: Some(tx_id), + lock_guard: Some(lock_guard), } } fn disarm(mut self) { - self.tx_id = None; - } -} - -impl Drop for GlobalTxAdmissionGuard<'_> { - fn drop(&mut self) { - if let Some(tx_id) = self.tx_id.take() { - self.module_host.abort_global_tx_locally(tx_id, true); + if let Some(lock_guard) = self.lock_guard.take() { + lock_guard.disarm(); } } } @@ -2111,21 +2103,27 @@ impl ModuleHost { } loop { - match manager.acquire(tx_id).await { - super::global_tx::AcquireDisposition::Acquired => { + match manager + .acquire(tx_id, |owner| async move { + if owner.creator_db != local_db { + self.send_wound_to_coordinator(owner).await; + } + }) + .await + { + super::global_tx::AcquireDisposition::Acquired(lock_guard) => { if let Some(outcome) = self.check_global_tx_wounded(tx_id) { self.abort_global_tx_locally(tx_id, true); return Err(outcome); } - return Ok(GlobalTxAdmissionGuard::new(self, tx_id)); + return Ok(GlobalTxAdmissionGuard::new(lock_guard)); } - super::global_tx::AcquireDisposition::Wound(owner) => { - let _ = manager.wound(&owner); - if owner.creator_db != local_db { - self.send_wound_to_coordinator(owner).await; - } + super::global_tx::AcquireDisposition::Cancelled => { + self.abort_global_tx_locally(tx_id, true); + return Err(self + .check_global_tx_wounded(tx_id) + .unwrap_or_else(|| ReducerOutcome::Wounded(Box::new(Box::from(Self::wounded_message(tx_id)))))); } - super::global_tx::AcquireDisposition::Wait => {} } } } @@ -2160,6 +2158,7 @@ impl ModuleHost { tx_component.parse().ok() } + // Notify a remote coordinator that a transaction should be wounded. async fn send_wound_to_coordinator(&self, tx_id: GlobalTxId) { let client = self.replica_ctx().call_reducer_client.clone(); let router = self.replica_ctx().call_reducer_router.clone(); @@ -2181,6 +2180,7 @@ impl ModuleHost { if let Some(token) = &auth_token { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } + log::info!("2PC wound: sending wound for {tx_id} to coordinator at {url}"); match req.send().await { Ok(resp) if resp.status().is_success() => { log::info!("2PC wound: notified coordinator for {tx_id}"); From 33ee98820b4ca1c27682580232d2e3101712aba3 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Mon, 30 Mar 2026 11:03:10 -0700 Subject: [PATCH 03/33] Fix some bugs and add some logs --- crates/client-api/src/routes/database.rs | 9 +- crates/core/src/client/client_connection.rs | 41 ++- crates/core/src/client/messages.rs | 4 +- crates/core/src/host/global_tx.rs | 282 ++++++++++++++++-- crates/core/src/host/instance_env.rs | 64 +++- crates/core/src/host/module_host.rs | 103 ++++--- crates/core/src/host/v8/mod.rs | 7 +- .../src/host/wasm_common/module_host_actor.rs | 147 ++++----- .../core/src/host/wasmtime/wasmtime_module.rs | 11 +- crates/core/src/replica_context.rs | 8 +- .../src/locking_tx_datastore/mut_tx.rs | 4 +- crates/lib/src/lib.rs | 2 +- crates/lib/src/tx_id.rs | 27 +- tools/tpcc-runner/src/loader.rs | 34 +-- 14 files changed, 531 insertions(+), 212 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index f0f4fa9da9c..d77a8662496 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -20,8 +20,8 @@ use axum::response::{ErrorResponse, IntoResponse}; use axum::routing::MethodRouter; use axum::Extension; use axum_extra::TypedHeader; -use http::HeaderMap; use futures::TryStreamExt; +use http::HeaderMap; use http::StatusCode; use log::{info, warn}; use serde::Deserialize; @@ -421,7 +421,12 @@ pub async fn wound_2pc( let tx_id = global_tx_id .parse::() .map_err(|e| (StatusCode::BAD_REQUEST, e).into_response())?; - let (module, _database) = find_module_and_database(&worker_ctx, name_or_identity).await?; + let (module, database) = find_module_and_database(&worker_ctx, name_or_identity).await?; + + log::info!( + "received 2PC wound request for transaction {tx_id} on database {}", + database.database_identity + ); module.wound_global_tx(tx_id).await.map_err(|e| { log::warn!("2PC wound failed for {tx_id}: {e}"); diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 21ff74ee822..a4c8dad4578 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -14,7 +14,7 @@ use crate::error::DBError; use crate::host::module_host::ClientConnectedError; use crate::host::{ CallProcedureReturn, FunctionArgs, ModuleHost, NoSuchModule, ProcedureCallResult, ReducerCallError, - ReducerCallResult, + ReducerCallResult, ReducerOutcome, }; use crate::subscription::module_subscription_manager::BroadcastError; use crate::subscription::row_list_builder_pool::JsonRowListBuilderFakePool; @@ -870,18 +870,33 @@ impl ClientConnection { timer: Instant, _flags: ws_v2::CallReducerFlags, ) -> Result { - self.module() - .call_reducer( - self.id.identity, - Some(self.id.connection_id), - None, - Some(self.sender()), - Some(request_id), - Some(timer), - reducer, - FunctionArgs::Bsatn(args), - ) - .await + const MAX_WOUNDED_RETRIES: usize = 3; + + let module = self.module(); + let mut tx_id = module.replica_ctx().mint_global_tx_id(Timestamp::now()); + + for attempt in 0..=MAX_WOUNDED_RETRIES { + let result = module + .call_reducer( + self.id.identity, + Some(self.id.connection_id), + Some(tx_id), + Some(self.sender()), + Some(request_id), + Some(timer), + reducer, + FunctionArgs::Bsatn(args.clone()), + ) + .await?; + + if !matches!(result.outcome, ReducerOutcome::Wounded(_)) || attempt == MAX_WOUNDED_RETRIES { + return Ok(result); + } + + tx_id = tx_id.next_attempt(); + } + + unreachable!("retry loop should return before exhausting attempts") } pub async fn call_procedure( diff --git a/crates/core/src/client/messages.rs b/crates/core/src/client/messages.rs index 18e88ddc3c5..f85eaac2410 100644 --- a/crates/core/src/client/messages.rs +++ b/crates/core/src/client/messages.rs @@ -381,9 +381,7 @@ impl ToProtocol for TransactionUpdateMessage { EventStatus::Committed(_) => ws_v1::UpdateStatus::Committed(update), EventStatus::FailedUser(errmsg) | EventStatus::FailedInternal(errmsg) - | EventStatus::Wounded(errmsg) => { - ws_v1::UpdateStatus::Failed(errmsg.clone().into()) - } + | EventStatus::Wounded(errmsg) => ws_v1::UpdateStatus::Failed(errmsg.clone().into()), EventStatus::OutOfEnergy => ws_v1::UpdateStatus::OutOfEnergy, }; diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index d593795d3ae..29b092bdcf7 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -1,6 +1,7 @@ use crate::identity::Identity; use spacetimedb_lib::GlobalTxId; -use std::collections::{HashMap, HashSet}; +use std::cmp::Ordering as CmpOrdering; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::future::Future; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -98,20 +99,54 @@ impl GlobalTxSession { struct LockState { owner: Option, - waiting: HashSet, + // An set of waiters ordered by tx_id with the oldest first. + waiting: BTreeSet, + // A map from wait_id to the corresponding wait entry, which contains the notify object to wake up the waiter when its turn comes. + wait_entries: HashMap, + waiter_ids_by_tx: HashMap, wounded_owners: HashSet, + next_wait_id: u64, } impl Default for LockState { fn default() -> Self { Self { owner: None, - waiting: HashSet::new(), + waiting: BTreeSet::new(), + wait_entries: HashMap::new(), + waiter_ids_by_tx: HashMap::new(), wounded_owners: HashSet::new(), + next_wait_id: 1, } } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct WaitKey { + tx_id: GlobalTxId, + wait_id: u64, +} + +impl Ord for WaitKey { + fn cmp(&self, other: &Self) -> CmpOrdering { + self.tx_id + .cmp(&other.tx_id) + .then_with(|| self.wait_id.cmp(&other.wait_id)) + } +} + +impl PartialOrd for WaitKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Debug)] +struct WaitEntry { + tx_id: GlobalTxId, + notify: Arc, +} + pub enum AcquireDisposition<'a> { Acquired(GlobalTxLockGuard<'a>), Cancelled, @@ -122,6 +157,37 @@ pub struct GlobalTxLockGuard<'a> { tx_id: Option, } +struct WaitRegistration<'a> { + manager: &'a GlobalTxManager, + wait_id: Option, +} + +impl<'a> WaitRegistration<'a> { + fn new(manager: &'a GlobalTxManager, wait_id: u64) -> Self { + Self { + manager, + wait_id: Some(wait_id), + } + } + + fn wait_id(&self) -> u64 { + self.wait_id.expect("registered waiter must still have a wait id") + } + + fn disarm(mut self) { + self.wait_id = None; + } +} + +impl Drop for WaitRegistration<'_> { + fn drop(&mut self) { + if let Some(wait_id) = self.wait_id.take() { + let mut ls = self.manager.lock_state.lock().unwrap(); + self.manager.remove_waiter_by_id(&mut ls, wait_id); + } + } +} + impl<'a> GlobalTxLockGuard<'a> { fn new(manager: &'a GlobalTxManager, tx_id: GlobalTxId) -> Self { Self { @@ -152,7 +218,6 @@ pub struct GlobalTxManager { sessions: Mutex>>, prepare_to_tx: Mutex>, lock_state: Mutex, - lock_notify: Notify, } impl GlobalTxManager { @@ -218,12 +283,21 @@ impl GlobalTxManager { self.get_session(tx_id).map(|s| s.subscribe_wounded()) } + // This should only be called by the coordinator. + // Arguably we should have a separate state for wounded and aborted, in case we wound a remote tx before we send write the prepare. pub fn wound(&self, tx_id: &GlobalTxId) -> Option> { let session = self.get_session(tx_id)?; - let _ = session.wound(); + let was_fresh = session.wound(); if !matches!(session.state(), GlobalTxState::Committed | GlobalTxState::Aborted) { session.set_state(GlobalTxState::Aborting); } + if was_fresh { + log::info!( + "global transaction {tx_id} marked wounded; role={:?} coordinator={}", + session.role, + session.coordinator_identity + ); + } Some(session) } @@ -236,46 +310,90 @@ impl GlobalTxManager { Some(rx) => rx, None => return AcquireDisposition::Cancelled, }; + let mut registration: Option> = None; loop { - // self.is_wounded(&tx_id) if *wounded_rx.borrow() { - self.remove_waiter(&tx_id); return AcquireDisposition::Cancelled; } - let (waiter, owner_to_wound) = { + let (notify, owner_to_wound, new_registration) = { let mut state = self.lock_state.lock().unwrap(); match state.owner { - None => { + None if self.is_next_waiter_locked(&state, tx_id) => { state.owner = Some(tx_id); - state.waiting.remove(&tx_id); + self.remove_waiter_locked(&mut state, &tx_id); + if let Some(registration) = registration.take() { + registration.disarm(); + } return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } + None => { + let (wait_id, notify) = if let Some(registration) = registration.as_ref() { + let wait_id = registration.wait_id(); + let notify = state + .wait_entries + .get(&wait_id) + .expect("wait entry must exist for registered waiter") + .notify + .clone(); + (wait_id, notify) + } else { + self.ensure_waiter_locked(&mut state, tx_id) + }; + let new_registration = registration.is_none().then(|| WaitRegistration::new(self, wait_id)); + (notify, None, new_registration) + } Some(owner) if owner == tx_id => { - state.waiting.remove(&tx_id); + self.remove_waiter_locked(&mut state, &tx_id); + if let Some(registration) = registration.take() { + registration.disarm(); + } return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } Some(owner) => { - state.waiting.insert(tx_id); + let (wait_id, notify) = if let Some(registration) = registration.as_ref() { + let wait_id = registration.wait_id(); + let notify = state + .wait_entries + .get(&wait_id) + .expect("wait entry must exist for registered waiter") + .notify + .clone(); + (wait_id, notify) + } else { + self.ensure_waiter_locked(&mut state, tx_id) + }; let owner_to_wound = (tx_id < owner && state.wounded_owners.insert(owner)).then_some(owner); - (self.lock_notify.notified(), owner_to_wound) + let new_registration = registration.is_none().then(|| WaitRegistration::new(self, wait_id)); + (notify, owner_to_wound, new_registration) } } }; + if let Some(new_registration) = new_registration { + registration = Some(new_registration); + } if let Some(owner) = owner_to_wound { - let _ = self.wound(&owner); + log::info!( + "global transaction {tx_id} is waiting behind younger owner {owner}; triggering wound flow" + ); + if self.should_wound_locally(&owner) { + let _ = self.wound(&owner); + } else { + log::info!( + "global transaction {tx_id} observed prepared participant owner {owner}; notifying coordinator without local wound" + ); + } on_wound(owner).await; } tokio::select! { changed = wounded_rx.changed(), if !*wounded_rx.borrow() => { if changed.is_ok() && *wounded_rx.borrow() { - self.remove_waiter(&tx_id); return AcquireDisposition::Cancelled; } } - _ = waiter => {} + _ = notify.notified() => {} } } } @@ -285,13 +403,79 @@ impl GlobalTxManager { if state.owner.as_ref() == Some(tx_id) { state.owner = None; state.wounded_owners.remove(tx_id); - self.lock_notify.notify_waiters(); + self.notify_next_waiter_locked(&state); + } else { + log::warn!("Release a lock that isn't actually held. This should not happen"); + } + self.remove_waiter_locked(&mut state, tx_id); + } + + fn ensure_waiter_locked(&self, state: &mut LockState, tx_id: GlobalTxId) -> (u64, Arc) { + if let Some(wait_id) = state.waiter_ids_by_tx.get(&tx_id).copied() { + let notify = state + .wait_entries + .get(&wait_id) + .expect("wait entry must exist for registered waiter") + .notify + .clone(); + return (wait_id, notify); } - state.waiting.remove(tx_id); + + let wait_id = state.next_wait_id; + state.next_wait_id += 1; + let notify = Arc::new(Notify::new()); + state.wait_entries.insert( + wait_id, + WaitEntry { + tx_id, + notify: notify.clone(), + }, + ); + state.waiter_ids_by_tx.insert(tx_id, wait_id); + state.waiting.insert(WaitKey { tx_id, wait_id }); + (wait_id, notify) } - fn remove_waiter(&self, tx_id: &GlobalTxId) { - self.lock_state.lock().unwrap().waiting.remove(tx_id); + fn is_next_waiter_locked(&self, state: &LockState, tx_id: GlobalTxId) -> bool { + match state.waiting.first() { + None => true, + Some(wait_key) => wait_key.tx_id == tx_id, + } + } + + fn notify_next_waiter_locked(&self, state: &LockState) { + if let Some(wait_key) = state.waiting.first() + && let Some(wait_entry) = state.wait_entries.get(&wait_key.wait_id) + { + wait_entry.notify.notify_one(); + } + } + + fn remove_waiter_locked(&self, state: &mut LockState, tx_id: &GlobalTxId) { + if let Some(wait_id) = state.waiter_ids_by_tx.remove(tx_id) { + state.wait_entries.remove(&wait_id); + state.waiting.remove(&WaitKey { tx_id: *tx_id, wait_id }); + } + } + + fn remove_waiter_by_id(&self, state: &mut LockState, wait_id: u64) { + let was_head = state.waiting.first().map(|w| w.wait_id) == Some(wait_id); + if let Some(wait_entry) = state.wait_entries.remove(&wait_id) { + state.waiter_ids_by_tx.remove(&wait_entry.tx_id); + state.waiting.remove(&WaitKey { + tx_id: wait_entry.tx_id, + wait_id, + }); + if was_head && state.owner.is_none() { + self.notify_next_waiter_locked(state); + } + } + } + + fn should_wound_locally(&self, tx_id: &GlobalTxId) -> bool { + self.get_session(tx_id) + .map(|session| !(session.role == GlobalTxRole::Participant && session.state() == GlobalTxState::Prepared)) + .unwrap_or(true) } } @@ -300,6 +484,7 @@ mod tests { use super::{AcquireDisposition, GlobalTxManager}; use crate::identity::Identity; use spacetimedb_lib::{GlobalTxId, Timestamp}; + use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Runtime; @@ -309,6 +494,7 @@ mod tests { Timestamp::from_micros_since_unix_epoch(ts), Identity::from_byte_array([db_byte; 32]), nonce, + 0, ) } @@ -317,11 +503,7 @@ mod tests { let manager = Arc::new(GlobalTxManager::default()); let younger = tx_id(20, 2, 0); let older = tx_id(10, 1, 0); - manager.ensure_session( - younger, - super::GlobalTxRole::Participant, - younger.creator_db, - ); + manager.ensure_session(younger, super::GlobalTxRole::Participant, younger.creator_db); manager.ensure_session(older, super::GlobalTxRole::Participant, older.creator_db); let rt = Runtime::new().unwrap(); @@ -340,10 +522,7 @@ mod tests { std::thread::sleep(Duration::from_millis(10)); assert!(manager.is_wounded(&younger)); drop(younger_guard); - assert!(matches!( - rt.block_on(older_task).expect("task should complete"), - true - )); + assert!(matches!(rt.block_on(older_task).expect("task should complete"), true)); } #[test] @@ -433,6 +612,46 @@ mod tests { }); } + #[test] + fn prepared_participant_only_signals_coordinator() { + let manager = Arc::new(GlobalTxManager::default()); + let owner = tx_id(20, 2, 0); + let older = tx_id(10, 1, 0); + let owner_session = manager.ensure_session(owner, super::GlobalTxRole::Participant, owner.creator_db); + owner_session.set_state(super::GlobalTxState::Prepared); + manager.ensure_session(older, super::GlobalTxRole::Participant, older.creator_db); + + let rt = Runtime::new().unwrap(); + let owner_guard = match rt.block_on(manager.acquire(owner, |_| async {})) { + AcquireDisposition::Acquired(guard) => guard, + AcquireDisposition::Cancelled => panic!("owner should acquire immediately"), + }; + + let coordinator_wounded = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = coordinator_wounded.clone(); + let manager_for_task = manager.clone(); + let older_task = rt.spawn(async move { + match manager_for_task + .acquire(older, move |_| { + let flag = flag.clone(); + async move { + flag.store(true, Ordering::SeqCst); + } + }) + .await + { + AcquireDisposition::Acquired(_guard) => true, + AcquireDisposition::Cancelled => false, + } + }); + + std::thread::sleep(Duration::from_millis(10)); + assert!(coordinator_wounded.load(Ordering::SeqCst)); + assert!(!manager.is_wounded(&owner)); + drop(owner_guard); + assert!(rt.block_on(older_task).expect("task should complete")); + } + #[test] fn wounded_waiter_is_cancelled() { let manager = Arc::new(GlobalTxManager::default()); @@ -458,9 +677,6 @@ mod tests { manager.wound(&waiter).expect("waiter session should exist"); drop(owner_guard); - assert!(matches!( - rt.block_on(waiter_task).expect("task should complete"), - true - )); + assert!(matches!(rt.block_on(waiter_task).expect("task should complete"), true)); } } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index cb787b7b66d..a9a6ce11656 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -35,7 +35,6 @@ use std::fmt::Display; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::ops::DerefMut; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::{Duration, Instant}; use std::vec::IntoIter; @@ -258,7 +257,13 @@ impl InstanceEnv { } /// Signal to this `InstanceEnv` that a function call is beginning. - pub fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType, tx_id: Option) { + pub fn start_funcall( + &mut self, + name: Identifier, + ts: Timestamp, + func_type: FuncCallType, + tx_id: Option, + ) { let is_reducer = matches!(func_type, FuncCallType::Reducer); self.start_time = ts; self.start_instant = Instant::now(); @@ -282,8 +287,7 @@ impl InstanceEnv { } fn mint_tx_id(&self, start_ts: Timestamp) -> GlobalTxId { - let nonce = self.replica_ctx.tx_id_nonce.fetch_add(1, Ordering::Relaxed); - GlobalTxId::new(start_ts, self.replica_ctx.database.database_identity, nonce) + self.replica_ctx.mint_global_tx_id(start_ts) } /// Returns the name of the most recent reducer to be run in this environment, @@ -1123,16 +1127,24 @@ impl InstanceEnv { .is_wounded(&tx_id) .then(|| self.wounded_tx_error(tx_id)) }); + let wounded_rx = tx_id.and_then(|tx_id| self.replica_ctx.global_tx_manager.subscribe_wounded(&tx_id)); + let wounded_message = + tx_id.map(|tx_id| format!("distributed transaction {tx_id} was wounded by an older transaction")); if let Some(tx_id) = tx_id { - let session = self - .replica_ctx - .global_tx_manager - .ensure_session(tx_id, GlobalTxRole::Coordinator, tx_id.creator_db); + let session = + self.replica_ctx + .global_tx_manager + .ensure_session(tx_id, GlobalTxRole::Coordinator, tx_id.creator_db); session.set_state(GlobalTxState::Preparing); } async move { + let tx_id = tx_id.ok_or_else(|| { + NodesError::HttpError( + "2PC remote reducer call requires an active distributed transaction id".to_owned(), + ) + })?; if let Some(err) = wounded_error { return Err(err); } @@ -1156,11 +1168,8 @@ impl InstanceEnv { if let Some(token) = auth_token { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } - if let Some(tx_id) = tx_id { - req = req.header(TX_ID_HEADER, tx_id.to_string()); - } - // TODO: This needs to select on subscribe_wounded as well, so we can stop waiting for the response when wounded. - let result = async { + req = req.header(TX_ID_HEADER, tx_id.to_string()); + let request_fut = async { let response = req.send().await.map_err(|e| NodesError::HttpError(e.to_string()))?; let status = response.status().as_u16(); let prepare_id = response @@ -1173,8 +1182,33 @@ impl InstanceEnv { .await .map_err(|e| NodesError::HttpError(e.to_string()))?; Ok((status, body, prepare_id)) - } - .await; + }; + tokio::pin!(request_fut); + let result = if let Some(mut wounded_rx) = wounded_rx { + tokio::select! { + result = &mut request_fut => result, + changed = wounded_rx.changed() => { + match changed { + Ok(()) if *wounded_rx.borrow() => { + log::info!( + "transaction {tx_id} was wounded during remote reducer call to {}/{}; aborting call", + database_identity.to_hex(), + reducer_name + ); + Err(NodesError::Wounded( + wounded_message + .clone() + .expect("wounded message should exist when tx_id exists"), + )) + }, + Ok(()) => request_fut.as_mut().await, + Err(_) => request_fut.as_mut().await, + } + } + } + } else { + request_fut.as_mut().await + }; WORKER_METRICS .cross_db_reducer_calls_total diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 0bf87089d35..15882348c5c 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1645,13 +1645,14 @@ impl ModuleHost { args, }; - let result = self.call( - &reducer_def.name, - call_reducer_params, - async |p, inst| Ok(inst.call_reducer(p)), - async |p, inst| inst.call_reducer(p).await, - ) - .await; + let result = self + .call( + &reducer_def.name, + call_reducer_params, + async |p, inst| Ok(inst.call_reducer(p)), + async |p, inst| inst.call_reducer(p).await, + ) + .await; if let Some(guard) = admission_guard { guard.disarm(); } @@ -1703,13 +1704,14 @@ impl ModuleHost { args, }; - let result = self.call( - &reducer_def.name, - call_reducer_params, - async |p, inst| Ok(inst.call_reducer_with_return(p)), - async |p, inst| inst.call_reducer(p).await.map(|res| (res, None)), - ) - .await; + let result = self + .call( + &reducer_def.name, + call_reducer_params, + async |p, inst| Ok(inst.call_reducer_with_return(p)), + async |p, inst| inst.call_reducer(p).await.map(|res| (res, None)), + ) + .await; if let Some(guard) = admission_guard { guard.disarm(); } @@ -1888,10 +1890,11 @@ impl ModuleHost { }, ); if let Some(tx_id) = tx_id { - let session = self - .replica_ctx() - .global_tx_manager - .ensure_session(tx_id, super::global_tx::GlobalTxRole::Participant, tx_id.creator_db); + let session = self.replica_ctx().global_tx_manager.ensure_session( + tx_id, + super::global_tx::GlobalTxRole::Participant, + tx_id.creator_db, + ); session.set_state(super::global_tx::GlobalTxState::Preparing); self.replica_ctx() .global_tx_manager @@ -2004,9 +2007,12 @@ impl ModuleHost { /// Abort a prepared transaction. pub fn abort_prepared(&self, prepare_id: &str) -> Result<(), String> { if let Some(tx_id) = self.replica_ctx().global_tx_manager.remove_prepare_mapping(prepare_id) { + log::info!("2PC abort_prepared: aborting prepared transaction {tx_id} ({prepare_id})"); self.replica_ctx() .global_tx_manager .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborting); + } else { + log::info!("2PC abort_prepared: aborting legacy/unmapped prepare_id {prepare_id}"); } let info = self .prepared_txs @@ -2027,14 +2033,26 @@ impl ModuleHost { } let Some(session) = self.replica_ctx().global_tx_manager.get_session(&tx_id) else { + log::info!("2PC wound: received wound for unknown transaction {tx_id} on {local_db}"); return Ok(()); }; match session.state() { super::global_tx::GlobalTxState::Committed | super::global_tx::GlobalTxState::Aborted - | super::global_tx::GlobalTxState::Aborting => return Ok(()), + | super::global_tx::GlobalTxState::Aborting => { + log::info!( + "2PC wound: transaction {tx_id} on {local_db} already in terminal/aborting state {:?}", + session.state() + ); + return Ok(()); + } _ => {} } + log::info!( + "2PC wound: coordinator {} is aborting transaction {tx_id} from state {:?}", + local_db, + session.state() + ); let session = self .replica_ctx() .global_tx_manager @@ -2078,9 +2096,7 @@ impl ModuleHost { ); } Err(e) => { - log::warn!( - "2PC wound: transport error aborting {prepare_id} on {participant_identity}: {e}" - ); + log::warn!("2PC wound: transport error aborting {prepare_id} on {participant_identity}: {e}"); } } } @@ -2098,32 +2114,33 @@ impl ModuleHost { }; manager.ensure_session(tx_id, role, tx_id.creator_db); if let Some(outcome) = self.check_global_tx_wounded(tx_id) { + log::info!("global transaction {tx_id} arrived already wounded before scheduler admission"); self.abort_global_tx_locally(tx_id, true); return Err(outcome); } - loop { - match manager - .acquire(tx_id, |owner| async move { - if owner.creator_db != local_db { - self.send_wound_to_coordinator(owner).await; - } - }) - .await - { - super::global_tx::AcquireDisposition::Acquired(lock_guard) => { - if let Some(outcome) = self.check_global_tx_wounded(tx_id) { - self.abort_global_tx_locally(tx_id, true); - return Err(outcome); - } - return Ok(GlobalTxAdmissionGuard::new(lock_guard)); + match manager + .acquire(tx_id, |owner| async move { + if owner.creator_db != local_db { + self.send_wound_to_coordinator(owner).await; } - super::global_tx::AcquireDisposition::Cancelled => { + }) + .await + { + super::global_tx::AcquireDisposition::Acquired(lock_guard) => { + if let Some(outcome) = self.check_global_tx_wounded(tx_id) { + log::info!("global transaction {tx_id} was wounded immediately after scheduler admission"); self.abort_global_tx_locally(tx_id, true); - return Err(self - .check_global_tx_wounded(tx_id) - .unwrap_or_else(|| ReducerOutcome::Wounded(Box::new(Box::from(Self::wounded_message(tx_id)))))); + return Err(outcome); } + return Ok(GlobalTxAdmissionGuard::new(lock_guard)); + } + super::global_tx::AcquireDisposition::Cancelled => { + log::info!("global transaction {tx_id} was cancelled while waiting for scheduler admission"); + self.abort_global_tx_locally(tx_id, true); + return Err(self + .check_global_tx_wounded(tx_id) + .unwrap_or_else(|| ReducerOutcome::Wounded(Box::new(Box::from(Self::wounded_message(tx_id)))))); } } } @@ -2136,6 +2153,10 @@ impl ModuleHost { } fn abort_global_tx_locally(&self, tx_id: GlobalTxId, remove_session: bool) { + log::info!( + "global transaction {tx_id} aborting locally on {}; remove_session={remove_session}", + self.replica_ctx().database.database_identity + ); self.replica_ctx() .global_tx_manager .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborted); diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 265f271d59e..4b4a72e0374 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -1446,12 +1446,7 @@ where // Start the timer. // We'd like this tightly around `call`. - env.start_funcall( - op.name().clone(), - op.timestamp(), - op.call_type(), - op.tx_id(), - ); + env.start_funcall(op.name().clone(), op.timestamp(), op.call_type(), op.tx_id()); v8::tc_scope!(scope, scope); let call_result = call(scope, op).map_err(|mut e| { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index faa21a48794..91f975666c5 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -4,6 +4,7 @@ use crate::client::ClientActorId; use crate::database_logger; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; +use crate::host::block_on_scoped; use crate::host::host_controller::CallProcedureReturn; use crate::host::instance_env::{InstanceEnv, TxSlot}; use crate::host::module_common::{build_common_module_from_raw, ModuleCommon}; @@ -13,7 +14,6 @@ use crate::host::module_host::{ ViewCallResult, ViewCommand, ViewCommandResult, ViewOutcome, }; use crate::host::scheduler::{CallScheduledFunctionResult, ScheduledFunctionParams}; -use crate::host::block_on_scoped; use crate::host::{ ArgsTuple, ModuleHost, ProcedureCallError, ProcedureCallResult, ReducerCallError, ReducerCallResult, ReducerId, ReducerOutcome, Scheduler, UpdateDatabaseResult, @@ -452,7 +452,8 @@ fn wounded_status(replica_ctx: &ReplicaContext, tx_id: spacetimedb_lib::GlobalTx } fn check_wounded(replica_ctx: &ReplicaContext, tx_id: Option) -> Option { - tx_id.filter(|tx_id| replica_ctx.global_tx_manager.is_wounded(tx_id)) + tx_id + .filter(|tx_id| replica_ctx.global_tx_manager.is_wounded(tx_id)) .map(|tx_id| wounded_status(replica_ctx, tx_id)) } @@ -723,12 +724,11 @@ impl WasmModuleInstance { // Step 3: wait for the PREPARE marker to be durable before signalling PREPARED. // B must not claim PREPARED until the marker is on disk — if B crashes after // claiming PREPARED but before the marker is durable, recovery has nothing to recover. - if let Some(prepare_offset) = marker_tx_data.tx_offset() { - if let Some(mut durable) = stdb.durable_tx_offset() { + if let Some(prepare_offset) = marker_tx_data.tx_offset() + && let Some(mut durable) = stdb.durable_tx_offset() { let handle = tokio::runtime::Handle::current(); let _ = block_on_scoped(&handle, durable.wait_for(prepare_offset)); } - } // Step 4: signal PREPARED. let res = ReducerCallResult { @@ -746,6 +746,12 @@ impl WasmModuleInstance { && Self::wait_for_2pc_decision(decision_rx, &prepare_id, coordinator_identity, &replica_ctx); if commit { + if let Some(tx_id) = global_tx_id { + log::info!( + "2PC participant {} committing prepared transaction {tx_id} ({prepare_id})", + replica_ctx.database.database_identity + ); + } // Delete the marker in the same tx as the reducer changes (atomic commit). if let Err(e) = tx.delete_st_2pc_state(&prepare_id) { log::error!("call_reducer_prepare_and_hold: failed to delete st_2pc_state for {prepare_id}: {e}"); @@ -786,6 +792,12 @@ impl WasmModuleInstance { replica_ctx.global_tx_manager.remove_session(&tx_id); } } else { + if let Some(tx_id) = global_tx_id { + log::info!( + "2PC participant {} aborting prepared transaction {tx_id} ({prepare_id})", + replica_ctx.database.database_identity + ); + } // ABORT: roll back reducer changes; clean up the already-committed marker. let _ = stdb.rollback_mut_tx(tx); if let Err(e) = stdb.with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |del_tx| { @@ -834,13 +846,16 @@ impl WasmModuleInstance { let auth_token = replica_ctx.call_reducer_auth_token.clone(); let prepare_id_owned = prepare_id.to_owned(); loop { - let decision = block_on_scoped(&handle, Self::query_coordinator_status( - &client, - &router, - auth_token.clone(), - coordinator_identity, - &prepare_id_owned, - )); + let decision = block_on_scoped( + &handle, + Self::query_coordinator_status( + &client, + &router, + auth_token.clone(), + coordinator_identity, + &prepare_id_owned, + ), + ); match decision { Some(commit) => return commit, None => std::thread::sleep(Duration::from_secs(5)), @@ -1182,7 +1197,7 @@ impl InstanceCommon { // crash recovery). if matches!(event.status, EventStatus::Committed(_)) && !prepared_participants.is_empty() { for (db_identity, prepare_id) in &prepared_participants { - if let Err(e) = tx.insert_st_2pc_coordinator_log(prepare_id, &db_identity.to_hex().to_string()) { + if let Err(e) = tx.insert_st_2pc_coordinator_log(prepare_id, db_identity.to_hex().as_ref()) { log::error!("insert_st_2pc_coordinator_log failed for {prepare_id}: {e}"); } } @@ -1207,64 +1222,62 @@ impl InstanceCommon { let replica_ctx = inst.replica_ctx().clone(); let handle = tokio::runtime::Handle::current(); block_on_scoped(&handle, async { - // Wait for A's coordinator log (committed atomically with the tx) to be - // durable before sending COMMIT to B. This guarantees that if A crashes - // after sending COMMIT, recovery can retransmit from the durable log. - if committed && let Some(mut durable_offset) = stdb.durable_tx_offset() { - if let Ok(offset) = commit_tx_offset.await { - let _ = durable_offset.wait_for(offset).await; - } - } + // Wait for A's coordinator log (committed atomically with the tx) to be + // durable before sending COMMIT to B. This guarantees that if A crashes + // after sending COMMIT, recovery can retransmit from the durable log. + if committed && let Some(mut durable_offset) = stdb.durable_tx_offset() + && let Ok(offset) = commit_tx_offset.await { + let _ = durable_offset.wait_for(offset).await; + } - let client = replica_ctx.call_reducer_client.clone(); - let router = replica_ctx.call_reducer_router.clone(); - let auth_token = replica_ctx.call_reducer_auth_token.clone(); - for (db_identity, prepare_id) in &prepared_participants { - let action = if committed { "commit" } else { "abort" }; - let base_url = match router.resolve_base_url(*db_identity).await { - Ok(url) => url, - Err(e) => { - log::error!("2PC {action}: failed to resolve base URL for {db_identity}: {e}"); - continue; + let client = replica_ctx.call_reducer_client.clone(); + let router = replica_ctx.call_reducer_router.clone(); + let auth_token = replica_ctx.call_reducer_auth_token.clone(); + for (db_identity, prepare_id) in &prepared_participants { + let action = if committed { "commit" } else { "abort" }; + let base_url = match router.resolve_base_url(*db_identity).await { + Ok(url) => url, + Err(e) => { + log::error!("2PC {action}: failed to resolve base URL for {db_identity}: {e}"); + continue; + } + }; + let url = format!( + "{}/v1/database/{}/2pc/{}/{}", + base_url, + db_identity.to_hex(), + action, + prepare_id, + ); + let mut req = client.post(&url); + if let Some(ref token) = auth_token { + req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + } + match req.send().await { + Ok(resp) if resp.status().is_success() => { + log::info!("2PC {action}: {prepare_id} on {db_identity}"); + // B acknowledged COMMIT — remove coordinator log entry + // (best-effort; recovery will clean up on restart if missed). + if committed + && let Err(e) = stdb + .with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |del_tx| { + Ok(del_tx.delete_st_2pc_coordinator_log(prepare_id)?) + }) + { + log::warn!("delete_st_2pc_coordinator_log failed for {prepare_id}: {e}"); } - }; - let url = format!( - "{}/v1/database/{}/2pc/{}/{}", - base_url, - db_identity.to_hex(), - action, - prepare_id, + } + Ok(resp) => { + log::error!( + "2PC {action}: failed for {prepare_id} on {db_identity}: status {}", + resp.status() ); - let mut req = client.post(&url); - if let Some(ref token) = auth_token { - req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); - } - match req.send().await { - Ok(resp) if resp.status().is_success() => { - log::info!("2PC {action}: {prepare_id} on {db_identity}"); - // B acknowledged COMMIT — remove coordinator log entry - // (best-effort; recovery will clean up on restart if missed). - if committed { - if let Err(e) = stdb - .with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |del_tx| { - Ok(del_tx.delete_st_2pc_coordinator_log(prepare_id)?) - }) - { - log::warn!("delete_st_2pc_coordinator_log failed for {prepare_id}: {e}"); - } - } - } - Ok(resp) => { - log::error!( - "2PC {action}: failed for {prepare_id} on {db_identity}: status {}", - resp.status() - ); - } - Err(e) => { - log::error!("2PC {action}: transport error for {prepare_id} on {db_identity}: {e}"); - } - } } + Err(e) => { + log::error!("2PC {action}: transport error for {prepare_id} on {db_identity}: {e}"); + } + } + } }); } diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 878a0be8b96..57388c2595f 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -582,10 +582,13 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let [conn_id_0, conn_id_1] = prepare_connection_id_for_call(op.caller_connection_id); // Prepare arguments to the reducer + the error sink & start timings. - let (args_source, result_sink) = - store - .data_mut() - .start_funcall(op.name.clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure, None); + let (args_source, result_sink) = store.data_mut().start_funcall( + op.name.clone(), + op.arg_bytes, + op.timestamp, + FuncCallType::Procedure, + None, + ); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { diff --git a/crates/core/src/replica_context.rs b/crates/core/src/replica_context.rs index aa190f13225..637b56b5382 100644 --- a/crates/core/src/replica_context.rs +++ b/crates/core/src/replica_context.rs @@ -7,9 +7,10 @@ use crate::host::global_tx::GlobalTxManager; use crate::host::reducer_router::ReducerCallRouter; use crate::messages::control_db::Database; use crate::subscription::module_subscription_actor::ModuleSubscriptions; +use spacetimedb_lib::{GlobalTxId, Timestamp}; use std::io; use std::ops::Deref; -use std::sync::atomic::AtomicU32; +use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -83,6 +84,11 @@ impl ReplicaContext { .build() .expect("failed to build call_reducer_on_db HTTP client") } + + pub fn mint_global_tx_id(&self, start_ts: Timestamp) -> GlobalTxId { + let nonce = self.tx_id_nonce.fetch_add(1, Ordering::Relaxed); + GlobalTxId::new(start_ts, self.database.database_identity, nonce, 0) + } } impl ReplicaContext { diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index b688c201054..cd2e3ae307d 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -2749,7 +2749,7 @@ impl MutTxId { /// Used on recovery: each row describes a transaction to resume. pub fn scan_st_2pc_state(&self) -> Result> { self.iter(ST_2PC_STATE_ID)? - .map(|row| St2pcStateRow::try_from(row)) + .map(St2pcStateRow::try_from) .collect() } @@ -2791,7 +2791,7 @@ impl MutTxId { /// Used on coordinator crash-recovery to retransmit COMMIT to participants. pub fn scan_st_2pc_coordinator_log(&self) -> Result> { self.iter(ST_2PC_COORDINATOR_LOG_ID)? - .map(|row| St2pcCoordinatorLogRow::try_from(row)) + .map(St2pcCoordinatorLogRow::try_from) .collect() } diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 7bbeb3d57fc..0f25a6fad87 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -39,7 +39,6 @@ pub use filterable_value::Private; pub use filterable_value::{FilterableValue, IndexScanRangeBoundsTerminator, TermBound}; pub use identity::Identity; pub use scheduler::ScheduleAt; -pub use tx_id::{GlobalTxId, TX_ID_HEADER}; pub use spacetimedb_sats::hash::{self, hash_bytes, Hash}; pub use spacetimedb_sats::time_duration::TimeDuration; pub use spacetimedb_sats::timestamp::Timestamp; @@ -49,6 +48,7 @@ pub use spacetimedb_sats::__make_register_reftype; pub use spacetimedb_sats::{self as sats, bsatn, buffer, de, ser}; pub use spacetimedb_sats::{AlgebraicType, ProductType, ProductTypeElement, SumType}; pub use spacetimedb_sats::{AlgebraicValue, ProductValue}; +pub use tx_id::{GlobalTxId, TX_ID_HEADER}; pub const MODULE_ABI_MAJOR_VERSION: u16 = 10; diff --git a/crates/lib/src/tx_id.rs b/crates/lib/src/tx_id.rs index 8e67829fbb9..7b6aa9072f3 100644 --- a/crates/lib/src/tx_id.rs +++ b/crates/lib/src/tx_id.rs @@ -8,22 +8,32 @@ pub const TX_ID_HEADER: &str = "X-Spacetime-Tx-Id"; /// A distributed reducer transaction identifier. /// /// Ordering is primarily by `start_ts`, so this can later support wound-wait. -/// `creator_db` namespaces the id globally, and `nonce` breaks ties for -/// multiple transactions started on the same database at the same timestamp. +/// `creator_db` namespaces the id globally, `nonce` breaks ties for +/// multiple transactions started on the same database at the same timestamp, +/// and `attempt` tracks retries of the same logical distributed transaction. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, SpacetimeType)] #[sats(crate = crate)] pub struct GlobalTxId { pub start_ts: Timestamp, pub creator_db: Identity, pub nonce: u32, + pub attempt: u32, } impl GlobalTxId { - pub const fn new(start_ts: Timestamp, creator_db: Identity, nonce: u32) -> Self { + pub const fn new(start_ts: Timestamp, creator_db: Identity, nonce: u32, attempt: u32) -> Self { Self { start_ts, creator_db, nonce, + attempt, + } + } + + pub const fn next_attempt(self) -> Self { + Self { + attempt: self.attempt + 1, + ..self } } } @@ -32,10 +42,11 @@ impl fmt::Display for GlobalTxId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "{}:{}:{:08x}", + "{}:{}:{:08x}:{:08x}", self.start_ts.to_micros_since_unix_epoch(), self.creator_db.to_hex(), - self.nonce + self.nonce, + self.attempt, ) } } @@ -44,10 +55,11 @@ impl FromStr for GlobalTxId { type Err = &'static str; fn from_str(s: &str) -> Result { - let mut parts = s.splitn(3, ':'); + let mut parts = s.splitn(4, ':'); let start_ts = parts.next().ok_or("missing tx timestamp")?; let creator_db = parts.next().ok_or("missing tx creator db")?; let nonce = parts.next().ok_or("missing tx nonce")?; + let attempt = parts.next().ok_or("missing tx attempt")?; if parts.next().is_some() { return Err("too many tx id components"); } @@ -58,7 +70,8 @@ impl FromStr for GlobalTxId { .map_err(|_| "invalid tx timestamp")?; let creator_db = Identity::from_hex(creator_db).map_err(|_| "invalid tx creator db")?; let nonce = u32::from_str_radix(nonce, 16).map_err(|_| "invalid tx nonce")?; + let attempt = u32::from_str_radix(attempt, 16).map_err(|_| "invalid tx attempt")?; - Ok(Self::new(start_ts, creator_db, nonce)) + Ok(Self::new(start_ts, creator_db, nonce, attempt)) } } diff --git a/tools/tpcc-runner/src/loader.rs b/tools/tpcc-runner/src/loader.rs index 96345ba8c81..0e08eef9b18 100644 --- a/tools/tpcc-runner/src/loader.rs +++ b/tools/tpcc-runner/src/loader.rs @@ -160,11 +160,11 @@ fn load_items( i_data: maybe_with_original(rng, 26, 50), }); if batch.len() >= batch_size { - client.queue_load_items(std::mem::take(&mut batch), &pending, &errors)?; + client.queue_load_items(std::mem::take(&mut batch), pending, errors)?; } } if !batch.is_empty() { - client.queue_load_items(batch, &pending, &errors)?; + client.queue_load_items(batch, pending, errors)?; } Ok(()) } @@ -205,7 +205,7 @@ fn load_remote_warehouses( let split_at = warehouse_batch.len().min(batch_size); let remainder = warehouse_batch.split_off(split_at); let rows = std::mem::replace(&mut warehouse_batch, remainder); - client.queue_load_remote_warehouses(rows, &pending, &errors)?; + client.queue_load_remote_warehouses(rows, pending, errors)?; } Ok(()) @@ -259,13 +259,13 @@ fn load_warehouses_and_districts( let split_at = warehouse_batch.len().min(batch_size); let remainder = warehouse_batch.split_off(split_at); let rows = std::mem::replace(&mut warehouse_batch, remainder); - client.queue_load_warehouses(rows, &pending, &errors)?; + client.queue_load_warehouses(rows, pending, errors)?; } while !district_batch.is_empty() { let split_at = district_batch.len().min(batch_size); let remainder = district_batch.split_off(split_at); let rows = std::mem::replace(&mut district_batch, remainder); - client.queue_load_districts(rows, &pending, &errors)?; + client.queue_load_districts(rows, pending, errors)?; } let _ = timestamp; Ok(()) @@ -304,12 +304,12 @@ fn load_stock( s_data: maybe_with_original(rng, 26, 50), }); if batch.len() >= batch_size { - client.queue_load_stocks(std::mem::take(&mut batch), &pending, &errors)?; + client.queue_load_stocks(std::mem::take(&mut batch), pending, errors)?; } } } if !batch.is_empty() { - client.queue_load_stocks(batch, &pending, &errors)?; + client.queue_load_stocks(batch, pending, errors)?; } Ok(()) } @@ -380,10 +380,10 @@ fn load_customers_history_orders( }); if customer_batch.len() >= batch_size { - client.queue_load_customers(std::mem::take(&mut customer_batch), &pending, &errors)?; + client.queue_load_customers(std::mem::take(&mut customer_batch), pending, errors)?; } if history_batch.len() >= batch_size { - client.queue_load_history(std::mem::take(&mut history_batch), &pending, &errors)?; + client.queue_load_history(std::mem::take(&mut history_batch), pending, errors)?; } } @@ -430,34 +430,34 @@ fn load_customers_history_orders( ol_dist_info: alpha_string(rng, 24, 24), }); if order_line_batch.len() >= batch_size { - client.queue_load_order_lines(std::mem::take(&mut order_line_batch), &pending, &errors)?; + client.queue_load_order_lines(std::mem::take(&mut order_line_batch), pending, errors)?; } } if order_batch.len() >= batch_size { - client.queue_load_orders(std::mem::take(&mut order_batch), &pending, &errors)?; + client.queue_load_orders(std::mem::take(&mut order_batch), pending, errors)?; } if new_order_batch.len() >= batch_size { - client.queue_load_new_orders(std::mem::take(&mut new_order_batch), &pending, &errors)?; + client.queue_load_new_orders(std::mem::take(&mut new_order_batch), pending, errors)?; } } } } if !customer_batch.is_empty() { - client.queue_load_customers(customer_batch, &pending, &errors)?; + client.queue_load_customers(customer_batch, pending, errors)?; } if !history_batch.is_empty() { - client.queue_load_history(history_batch, &pending, &errors)?; + client.queue_load_history(history_batch, pending, errors)?; } if !order_batch.is_empty() { - client.queue_load_orders(order_batch, &pending, &errors)?; + client.queue_load_orders(order_batch, pending, errors)?; } if !new_order_batch.is_empty() { - client.queue_load_new_orders(new_order_batch, &pending, &errors)?; + client.queue_load_new_orders(new_order_batch, pending, errors)?; } if !order_line_batch.is_empty() { - client.queue_load_order_lines(order_line_batch, &pending, &errors)?; + client.queue_load_order_lines(order_line_batch, pending, errors)?; } Ok(()) From 17d3e5f192312a17623208eb4f09db6fd76d6d47 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Mon, 30 Mar 2026 12:39:36 -0700 Subject: [PATCH 04/33] Add more retries and backoff --- crates/core/src/client/client_connection.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index a4c8dad4578..efe7dfc2d0f 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -23,6 +23,7 @@ use crate::util::prometheus_handle::IntGaugeExt; use crate::worker_metrics::WORKER_METRICS; use bytes::Bytes; use bytestring::ByteString; +use std::time::Duration; use derive_more::From; use futures::prelude::*; use prometheus::{Histogram, IntCounter, IntGauge}; @@ -35,6 +36,7 @@ use spacetimedb_lib::{bsatn, Identity, TimeDuration, Timestamp}; use tokio::sync::mpsc::error::{SendError, TrySendError}; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::AbortHandle; +use tokio::time::sleep; use tracing::{trace, warn}; #[derive(PartialEq, Eq, Clone, Copy, Hash, Debug)] @@ -870,10 +872,12 @@ impl ClientConnection { timer: Instant, _flags: ws_v2::CallReducerFlags, ) -> Result { - const MAX_WOUNDED_RETRIES: usize = 3; + const MAX_WOUNDED_RETRIES: usize = 10; + const MAX_BACKOFF: Duration = Duration::from_millis(100); let module = self.module(); let mut tx_id = module.replica_ctx().mint_global_tx_id(Timestamp::now()); + let mut wound_backoff = Duration::from_millis(10); for attempt in 0..=MAX_WOUNDED_RETRIES { let result = module @@ -890,8 +894,15 @@ impl ClientConnection { .await?; if !matches!(result.outcome, ReducerOutcome::Wounded(_)) || attempt == MAX_WOUNDED_RETRIES { + if attempt == MAX_WOUNDED_RETRIES && matches!(result.outcome, ReducerOutcome::Wounded(_)) { + log::warn!("Reducer call was wounded on final attempt. Returning error to client."); + } return Ok(result); } + + log::info!("Reducer call was wounded on attempt {attempt}, retrying after {wound_backoff:?} with new transaction ID {tx_id}"); + sleep(wound_backoff).await; + wound_backoff = wound_backoff.mul_f32(2.0).min(MAX_BACKOFF); tx_id = tx_id.next_attempt(); } From 85bc884ffb616517f14cb05ad1ca497fec2b5fa3 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 01:47:20 +0530 Subject: [PATCH 05/33] point to 2pc bindings --- crates/bindings/src/remote_reducer.rs | 6 ++++-- modules/tpcc/src/remote.rs | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/bindings/src/remote_reducer.rs b/crates/bindings/src/remote_reducer.rs index 701282c66f6..cbd18fea05e 100644 --- a/crates/bindings/src/remote_reducer.rs +++ b/crates/bindings/src/remote_reducer.rs @@ -103,12 +103,14 @@ pub fn call_reducer_on_db_2pc( database_identity: Identity, reducer_name: &str, args: &[u8], -) -> Result<(), RemoteCallError> { +) -> Result, RemoteCallError> { let identity_bytes = database_identity.to_byte_array(); match spacetimedb_bindings_sys::call_reducer_on_db_2pc(identity_bytes, reducer_name, args) { Ok((status, body_source)) => { if status < 300 { - return Ok(()); + let mut out = Vec::new(); + read_bytes_source_into(body_source, &mut out); + return Ok(out); } let msg = if body_source == spacetimedb_bindings_sys::raw::BytesSource::INVALID { String::new() diff --git a/modules/tpcc/src/remote.rs b/modules/tpcc/src/remote.rs index e50ba08c1cd..bd7c9a406d3 100644 --- a/modules/tpcc/src/remote.rs +++ b/modules/tpcc/src/remote.rs @@ -1,10 +1,9 @@ use spacetimedb::{ - reducer, remote_reducer::call_reducer_on_db, table, DeserializeOwned, Identity, ReducerContext, Serialize, + reducer, remote_reducer::call_reducer_on_db_2pc, table, DeserializeOwned, Identity, ReducerContext, Serialize, SpacetimeType, Table, }; use spacetimedb_sats::bsatn; - /// For warehouses not managed by this database, stores the [`Identity`] of the remote database which manages that warehouse. /// /// Will not have a row present for a warehouse managed by the local database. @@ -55,7 +54,7 @@ where let args = bsatn::to_vec(args).map_err(|e| { format!("Failed to BSATN-serialize args for remote reducer {reducer_name} on database {database_ident}: {e}") })?; - let out = call_reducer_on_db(database_ident, reducer_name, &args) + let out = call_reducer_on_db_2pc(database_ident, reducer_name, &args) .map_err(|e| format!("Failed to call remote reducer {reducer_name} on database {database_ident}: {e}"))?; bsatn::from_slice(&out).map_err(|e| { format!( From 4d32126bebfc45f41132d942788017743d0d8c5f Mon Sep 17 00:00:00 2001 From: Alessandro Asoni Date: Mon, 30 Mar 2026 22:19:34 +0200 Subject: [PATCH 06/33] Add extra time spans --- modules/tpcc/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/tpcc/src/lib.rs b/modules/tpcc/src/lib.rs index c0c83e5965b..51e63423ff1 100644 --- a/modules/tpcc/src/lib.rs +++ b/modules/tpcc/src/lib.rs @@ -492,11 +492,14 @@ pub fn stock_level( ) -> Result { let _timer = LogStopwatch::new("stock_level"); + let _timer_district = LogStopwatch::new("stock_level_district"); let district = find_district(ctx, w_id, d_id)?; + _timer_district.end(); let start_o_id = district.d_next_o_id.saturating_sub(20); let end_o_id = district.d_next_o_id; let mut item_ids = BTreeSet::new(); + let _timer_filter = LogStopwatch::new("stock_level_filter"); for line in ctx .db .order_line() @@ -505,14 +508,17 @@ pub fn stock_level( { item_ids.insert(line.ol_i_id); } + _timer_filter.end(); let mut low_stock_count = 0u32; + let _timer_count= LogStopwatch::new("stock_level_count"); for item_id in item_ids { let stock = find_stock(ctx, w_id, item_id)?; if stock.s_quantity < threshold { low_stock_count += 1; } } + _timer_count.end(); Ok(StockLevelResult { warehouse_id: w_id, From 075c055ebd5aad3745d11effba1a89e4200274ad Mon Sep 17 00:00:00 2001 From: Alessandro Asoni Date: Mon, 30 Mar 2026 22:49:29 +0200 Subject: [PATCH 07/33] performance improvement for stock_level --- modules/tpcc/src/lib.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/modules/tpcc/src/lib.rs b/modules/tpcc/src/lib.rs index 51e63423ff1..d1e882effd4 100644 --- a/modules/tpcc/src/lib.rs +++ b/modules/tpcc/src/lib.rs @@ -181,7 +181,10 @@ pub struct Item { pub i_data: String, } -#[table(accessor = stock)] +#[table( + accessor = stock, + index(accessor = by_stock_key_quantity, btree(columns = [stock_key, s_quantity])), +)] #[derive(Clone, Debug)] pub struct Stock { #[primary_key] @@ -513,8 +516,7 @@ pub fn stock_level( let mut low_stock_count = 0u32; let _timer_count= LogStopwatch::new("stock_level_count"); for item_id in item_ids { - let stock = find_stock(ctx, w_id, item_id)?; - if stock.s_quantity < threshold { + if find_low_stock(ctx, w_id, item_id, threshold).is_some() { low_stock_count += 1; } } @@ -792,6 +794,15 @@ fn find_customer_by_id(tx: &ReducerContext, w_id: WarehouseId, d_id: u8, c_id: u .ok_or_else(|| format!("customer ({w_id}, {d_id}, {c_id}) not found")) } +fn find_low_stock(tx: &ReducerContext, w_id: WarehouseId, item_id: u32, threshold: i32) -> Option { + let stock_key = pack_stock_key(w_id, item_id); + tx.db + .stock() + .by_stock_key_quantity() + .filter((stock_key, 0..threshold)) + .next() +} + fn find_stock(tx: &ReducerContext, w_id: WarehouseId, item_id: u32) -> Result { let stock_key = pack_stock_key(w_id, item_id); tx.db From deaf41e824cf1b085c2789ab7ae8ff0ce832a56f Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Mon, 30 Mar 2026 15:40:54 -0700 Subject: [PATCH 08/33] Vibe fix to make wounded errors bubble up --- crates/bindings-sys/src/lib.rs | 20 ++++++------- crates/bindings/src/remote_reducer.rs | 28 ++++++++++++++++--- crates/bindings/src/rt.rs | 6 ++++ crates/core/src/host/v8/mod.rs | 1 + crates/core/src/host/wasm_common.rs | 2 +- .../src/host/wasm_common/module_host_actor.rs | 6 ++++ .../src/host/wasmtime/wasm_instance_env.rs | 20 +++++++++++-- .../core/src/host/wasmtime/wasmtime_module.rs | 14 +++++++++- crates/primitives/src/errno.rs | 1 + modules/tpcc/src/remote.rs | 11 +++++--- 10 files changed, 87 insertions(+), 22 deletions(-) diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 927c444a38d..24b355983c0 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -1504,7 +1504,7 @@ pub fn call_reducer_on_db( identity: [u8; 32], reducer_name: &str, args: &[u8], -) -> Result<(u16, raw::BytesSource), raw::BytesSource> { +) -> Result<(u16, raw::BytesSource), (Errno, raw::BytesSource)> { let mut out = raw::BytesSource::INVALID; let status = unsafe { raw::call_reducer_on_db( @@ -1520,10 +1520,10 @@ pub fn call_reducer_on_db( // on transport failure. Unlike other ABI functions, a non-zero return value here // does NOT indicate a generic errno — it's the HTTP status code. Only HTTP_ERROR // specifically signals a transport-level failure. - if status == Errno::HTTP_ERROR.code() { - Err(out) - } else { - Ok((status, out)) + match Errno::from_code(status) { + Some(errno @ (Errno::HTTP_ERROR | Errno::WOUNDED_TRANSACTION)) => Err((errno, out)), + Some(errno) => panic!("{errno:?}"), + None => Ok((status, out)), } } @@ -1539,7 +1539,7 @@ pub fn call_reducer_on_db_2pc( identity: [u8; 32], reducer_name: &str, args: &[u8], -) -> Result<(u16, raw::BytesSource), raw::BytesSource> { +) -> Result<(u16, raw::BytesSource), (Errno, raw::BytesSource)> { let mut out = raw::BytesSource::INVALID; let status = unsafe { raw::call_reducer_on_db_2pc( @@ -1551,10 +1551,10 @@ pub fn call_reducer_on_db_2pc( &mut out, ) }; - if status == Errno::HTTP_ERROR.code() { - Err(out) - } else { - Ok((status, out)) + match Errno::from_code(status) { + Some(errno @ (Errno::HTTP_ERROR | Errno::WOUNDED_TRANSACTION)) => Err((errno, out)), + Some(errno) => panic!("{errno:?}"), + None => Ok((status, out)), } } diff --git a/crates/bindings/src/remote_reducer.rs b/crates/bindings/src/remote_reducer.rs index 701282c66f6..1e1c3549934 100644 --- a/crates/bindings/src/remote_reducer.rs +++ b/crates/bindings/src/remote_reducer.rs @@ -19,6 +19,7 @@ //! Err(remote_reducer::RemoteCallError::Failed(msg)) => log::error!("reducer failed: {msg}"), //! Err(remote_reducer::RemoteCallError::NotFound(msg)) => log::error!("not found: {msg}"), //! Err(remote_reducer::RemoteCallError::Unreachable(msg)) => log::error!("unreachable: {msg}"), +//! Err(remote_reducer::RemoteCallError::Wounded(msg)) => log::warn!("wounded: {msg}"), //! } //! } //! ``` @@ -34,6 +35,8 @@ pub enum RemoteCallError { NotFound(String), /// The call could not be delivered (connection refused, timeout, network error, etc.). Unreachable(String), + /// The distributed transaction was wounded by an older transaction. + Wounded(String), } impl core::fmt::Display for RemoteCallError { @@ -42,10 +45,18 @@ impl core::fmt::Display for RemoteCallError { RemoteCallError::Failed(msg) => write!(f, "remote reducer failed: {msg}"), RemoteCallError::NotFound(msg) => write!(f, "remote database or reducer not found: {msg}"), RemoteCallError::Unreachable(msg) => write!(f, "remote database unreachable: {msg}"), + RemoteCallError::Wounded(msg) => write!(f, "{msg}"), } } } +pub fn into_reducer_error_message(error: RemoteCallError) -> String { + match error { + RemoteCallError::Wounded(msg) => crate::rt::encode_wounded_error_message(msg), + other => other.to_string(), + } +} + /// Call a reducer on a remote database. /// /// - `database_identity`: the target database. @@ -56,6 +67,7 @@ impl core::fmt::Display for RemoteCallError { /// Returns `Err(RemoteCallError::Failed(msg))` when the reducer ran but returned an error. /// Returns `Err(RemoteCallError::NotFound(msg))` when the database or reducer does not exist. /// Returns `Err(RemoteCallError::Unreachable(msg))` on transport failure (connection refused, timeout, …). +/// Returns `Err(RemoteCallError::Wounded(msg))` if the surrounding distributed transaction was wounded. pub fn call_reducer_on_db( database_identity: Identity, reducer_name: &str, @@ -83,10 +95,14 @@ pub fn call_reducer_on_db( Err(RemoteCallError::Failed(msg)) } } - Err(err_source) => { + Err((errno, err_source)) => { use crate::rt::read_bytes_source_as; let msg = read_bytes_source_as::(err_source); - Err(RemoteCallError::Unreachable(msg)) + Err(if errno == spacetimedb_bindings_sys::Errno::WOUNDED_TRANSACTION { + RemoteCallError::Wounded(msg) + } else { + RemoteCallError::Unreachable(msg) + }) } } } @@ -123,10 +139,14 @@ pub fn call_reducer_on_db_2pc( Err(RemoteCallError::Failed(msg)) } } - Err(err_source) => { + Err((errno, err_source)) => { use crate::rt::read_bytes_source_as; let msg = read_bytes_source_as::(err_source); - Err(RemoteCallError::Unreachable(msg)) + Err(if errno == spacetimedb_bindings_sys::Errno::WOUNDED_TRANSACTION { + RemoteCallError::Wounded(msg) + } else { + RemoteCallError::Unreachable(msg) + }) } } } diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 780e514e7dd..24a7e2aff3f 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -919,6 +919,12 @@ static DESCRIBERS: Mutex>> = Mutex::new(Vec::new()); /// A reducer function takes in `(ReducerContext, Args)` /// and returns a result with a possible error message. pub type ReducerFn = fn(&ReducerContext, &[u8]) -> ReducerResult; + +const WOUNDED_ERROR_PREFIX: &str = "__STDB_WOUNDED__:"; + +pub fn encode_wounded_error_message(message: impl Into) -> String { + format!("{WOUNDED_ERROR_PREFIX}{}", message.into()) +} static REDUCERS: OnceLock> = OnceLock::new(); #[cfg(feature = "unstable")] diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 4b4a72e0374..6affa7d5386 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -1420,6 +1420,7 @@ impl WasmInstance for V8Instance<'_, '_, '_> { .map_result(|call_result| { call_result.map_err(|e| match e { ExecutionError::User(e) => anyhow::Error::msg(e), + ExecutionError::Wounded(e) => anyhow::Error::msg(e), ExecutionError::Recoverable(e) | ExecutionError::Trap(e) => e, }) }); diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 30e6689e532..591ef5a8df5 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -357,7 +357,7 @@ pub fn err_to_errno(err: NodesError) -> Result<(NonZeroU16, Option), Nod NodesError::IndexRowNotFound => errno::NO_SUCH_ROW, NodesError::IndexCannotSeekRange => errno::WRONG_INDEX_ALGO, NodesError::ScheduleError(ScheduleError::DelayTooLong(_)) => errno::SCHEDULE_AT_DELAY_TOO_LONG, - NodesError::Wounded(message) => return Ok((errno::HTTP_ERROR, Some(message))), + NodesError::Wounded(message) => return Ok((errno::WOUNDED_TRANSACTION, Some(message))), NodesError::HttpError(message) => return Ok((errno::HTTP_ERROR, Some(message))), NodesError::Internal(ref internal) => match **internal { DBError::Datastore(DatastoreError::Index(IndexError::UniqueConstraintViolation( diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 78d067cce3e..b89373f4f96 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -252,6 +252,7 @@ impl ExecutionStats { pub enum ExecutionError { User(Box), + Wounded(Box), Recoverable(anyhow::Error), Trap(anyhow::Error), } @@ -1405,6 +1406,7 @@ impl InstanceCommon { ); (EventStatus::FailedUser(err.into()), None) } + Err(ExecutionError::Wounded(err)) => (EventStatus::Wounded(err.into()), None), // We haven't actually committed yet - `commit_and_broadcast_event` will commit // for us and replace this with the actual database update. Ok(return_value) => { @@ -1730,6 +1732,10 @@ impl InstanceCommon { inst.log_traceback("view", &view_name, &anyhow::anyhow!(err)); self.handle_outer_error(&result.stats.energy, &view_name).into() } + (Err(ExecutionError::Wounded(err)), _) => { + inst.log_traceback("view", &view_name, &anyhow::anyhow!(err)); + self.handle_outer_error(&result.stats.energy, &view_name).into() + } (Ok(raw), sender) => { // This is wrapped in a closure to simplify error handling. let outcome: Result = (|| { diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 6f59a46471f..638b2777d13 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -2009,7 +2009,7 @@ impl WasmInstanceEnv { bytes_source.0.write_to(mem, out)?; Ok(status as u32) } - Err(NodesError::HttpError(err) | NodesError::Wounded(err)) => { + Err(NodesError::HttpError(err)) => { let err_bytes = bsatn::to_vec(&err).with_context(|| { format!("Failed to BSATN-serialize call_reducer_on_db transport error: {err:?}") })?; @@ -2017,6 +2017,14 @@ impl WasmInstanceEnv { bytes_source.0.write_to(mem, out)?; Ok(errno::HTTP_ERROR.get() as u32) } + Err(NodesError::Wounded(err)) => { + let err_bytes = bsatn::to_vec(&err).with_context(|| { + format!("Failed to BSATN-serialize call_reducer_on_db wounded error: {err:?}") + })?; + let bytes_source = WasmInstanceEnv::create_bytes_source(env, err_bytes.into())?; + bytes_source.0.write_to(mem, out)?; + Ok(errno::WOUNDED_TRANSACTION.get() as u32) + } Err(e) => Err(WasmError::Db(e)), } }) @@ -2082,7 +2090,7 @@ impl WasmInstanceEnv { bytes_source.0.write_to(mem, out)?; Ok(status as u32) } - Err(NodesError::HttpError(err) | NodesError::Wounded(err)) => { + Err(NodesError::HttpError(err)) => { let err_bytes = bsatn::to_vec(&err).with_context(|| { format!("Failed to BSATN-serialize call_reducer_on_db_2pc transport error: {err:?}") })?; @@ -2090,6 +2098,14 @@ impl WasmInstanceEnv { bytes_source.0.write_to(mem, out)?; Ok(errno::HTTP_ERROR.get() as u32) } + Err(NodesError::Wounded(err)) => { + let err_bytes = bsatn::to_vec(&err).with_context(|| { + format!("Failed to BSATN-serialize call_reducer_on_db_2pc wounded error: {err:?}") + })?; + let bytes_source = WasmInstanceEnv::create_bytes_source(env, err_bytes.into())?; + bytes_source.0.write_to(mem, out)?; + Ok(errno::WOUNDED_TRANSACTION.get() as u32) + } Err(e) => Err(WasmError::Db(e)), } }) diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 57388c2595f..4d3b71b43d4 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -107,6 +107,17 @@ pub(super) enum ViewResultSinkError { UnexpectedCode(i32), } +const WOUNDED_ERROR_PREFIX: &str = "__STDB_WOUNDED__:"; + +fn decode_reducer_failure(result: Vec) -> ExecutionError { + let message = string_from_utf8_lossy_owned(result); + if let Some(message) = message.strip_prefix(WOUNDED_ERROR_PREFIX) { + ExecutionError::Wounded(message.into()) + } else { + ExecutionError::User(message.into()) + } +} + /// Handle the return code from a function using a result sink. /// /// On success, returns the result bytes. @@ -114,7 +125,7 @@ pub(super) enum ViewResultSinkError { fn handle_result_sink_code(code: i32, result: Vec) -> Result, ExecutionError> { match code { 0 => Ok(result), - CALL_FAILURE => Err(ExecutionError::User(string_from_utf8_lossy_owned(result).into())), + CALL_FAILURE => Err(decode_reducer_failure(result)), _ => Err(ExecutionError::Recoverable(anyhow::anyhow!("unknown return code"))), } } @@ -253,6 +264,7 @@ impl module_host_actor::WasmInstancePre for WasmtimeModule { res.map_err(|e| match e { ExecutionError::User(err) => InitializationError::Setup(err), + ExecutionError::Wounded(err) => InitializationError::Setup(err), ExecutionError::Recoverable(err) | ExecutionError::Trap(err) => { let func = SETUP_DUNDER.to_owned(); InitializationError::RuntimeError { err, func } diff --git a/crates/primitives/src/errno.rs b/crates/primitives/src/errno.rs index 5c422941715..8221e5a447c 100644 --- a/crates/primitives/src/errno.rs +++ b/crates/primitives/src/errno.rs @@ -35,6 +35,7 @@ macro_rules! errnos { "ABI call can only be made while within a read-only transaction" ), HTTP_ERROR(21, "The HTTP request failed"), + WOUNDED_TRANSACTION(22, "The distributed transaction was wounded by an older transaction"), ); }; } diff --git a/modules/tpcc/src/remote.rs b/modules/tpcc/src/remote.rs index e50ba08c1cd..52b757af3f4 100644 --- a/modules/tpcc/src/remote.rs +++ b/modules/tpcc/src/remote.rs @@ -1,6 +1,7 @@ use spacetimedb::{ - reducer, remote_reducer::call_reducer_on_db, table, DeserializeOwned, Identity, ReducerContext, Serialize, - SpacetimeType, Table, + reducer, + remote_reducer::{call_reducer_on_db, into_reducer_error_message, RemoteCallError}, + table, DeserializeOwned, Identity, ReducerContext, Serialize, SpacetimeType, Table, }; use spacetimedb_sats::bsatn; @@ -55,8 +56,10 @@ where let args = bsatn::to_vec(args).map_err(|e| { format!("Failed to BSATN-serialize args for remote reducer {reducer_name} on database {database_ident}: {e}") })?; - let out = call_reducer_on_db(database_ident, reducer_name, &args) - .map_err(|e| format!("Failed to call remote reducer {reducer_name} on database {database_ident}: {e}"))?; + let out = call_reducer_on_db(database_ident, reducer_name, &args).map_err(|e| match e { + RemoteCallError::Wounded(_) => into_reducer_error_message(e), + _ => format!("Failed to call remote reducer {reducer_name} on database {database_ident}: {e}"), + })?; bsatn::from_slice(&out).map_err(|e| { format!( "Failed to BSATN-deserialize result from remote reducer {reducer_name} on database {database_ident}: {e}" From 3ef00b3b5167de35e02768d7f20ae26750a5e758 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Mon, 30 Mar 2026 16:36:33 -0700 Subject: [PATCH 09/33] Add configurable grace period before wounding --- crates/core/src/config.rs | 55 +++++++++++ crates/core/src/host/global_tx.rs | 109 +++++++++++++++++++-- crates/core/src/host/host_controller.rs | 17 +++- crates/core/src/host/module_host.rs | 10 +- crates/standalone/config.toml | 4 + crates/standalone/src/lib.rs | 5 +- crates/standalone/src/subcommands/start.rs | 1 + 7 files changed, 185 insertions(+), 16 deletions(-) diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 5eec2eaf4e3..0d57890fc49 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -136,6 +136,8 @@ pub struct ConfigFile { pub logs: LogConfig, #[serde(default)] pub v8_heap_policy: V8HeapPolicyConfig, + #[serde(default)] + pub global_tx: GlobalTxConfig, } impl ConfigFile { @@ -189,6 +191,21 @@ pub struct V8HeapPolicyConfig { pub heap_limit_bytes: Option, } +#[derive(Clone, Copy, Debug, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct GlobalTxConfig { + #[serde(default = "default_wound_grace_period", deserialize_with = "de_duration")] + pub wound_grace_period: Duration, +} + +impl Default for GlobalTxConfig { + fn default() -> Self { + Self { + wound_grace_period: default_wound_grace_period(), + } + } +} + impl Default for V8HeapPolicyConfig { fn default() -> Self { Self { @@ -237,6 +254,10 @@ fn def_retire() -> f64 { 0.75 } +fn default_wound_grace_period() -> Duration { + Duration::from_millis(10) +} + fn de_nz_u64<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, @@ -264,6 +285,23 @@ where Ok((!duration.is_zero()).then_some(duration)) } +fn de_duration<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + #[derive(serde::Deserialize)] + #[serde(untagged)] + enum DurationValue { + String(String), + Seconds(u64), + } + + match DurationValue::deserialize(deserializer)? { + DurationValue::String(value) => humantime::parse_duration(&value).map_err(serde::de::Error::custom), + DurationValue::Seconds(value) => Ok(Duration::from_secs(value)), + } +} + fn de_fraction<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -445,4 +483,21 @@ mod tests { assert_eq!(config.v8_heap_policy.heap_retire_fraction, 0.8); assert_eq!(config.v8_heap_policy.heap_limit_bytes, Some(256 * 1024 * 1024)); } + + #[test] + fn global_tx_defaults_when_omitted() { + let config: ConfigFile = toml::from_str("").unwrap(); + assert_eq!(config.global_tx.wound_grace_period, Duration::from_millis(10)); + } + + #[test] + fn global_tx_parses_from_toml() { + let toml = r#" + [global-tx] + wound-grace-period = "25ms" + "#; + + let config: ConfigFile = toml::from_str(toml).unwrap(); + assert_eq!(config.global_tx.wound_grace_period, Duration::from_millis(25)); + } } diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index 29b092bdcf7..84e3fb17b49 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -5,8 +5,11 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::future::Future; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; use tokio::sync::{watch, Notify}; +const DEFAULT_WOUND_GRACE_PERIOD: Duration = Duration::from_millis(10); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GlobalTxRole { Coordinator, @@ -213,14 +216,33 @@ impl Drop for GlobalTxLockGuard<'_> { } } -#[derive(Default)] pub struct GlobalTxManager { sessions: Mutex>>, prepare_to_tx: Mutex>, lock_state: Mutex, + wound_grace_period: Duration, +} + +impl Default for GlobalTxManager { + fn default() -> Self { + Self::new(DEFAULT_WOUND_GRACE_PERIOD) + } } impl GlobalTxManager { + pub fn new(wound_grace_period: Duration) -> Self { + Self { + sessions: Mutex::default(), + prepare_to_tx: Mutex::default(), + lock_state: Mutex::default(), + wound_grace_period, + } + } + + pub fn wound_grace_period(&self) -> Duration { + self.wound_grace_period + } + pub fn ensure_session( &self, tx_id: GlobalTxId, @@ -304,7 +326,7 @@ impl GlobalTxManager { pub async fn acquire(&self, tx_id: GlobalTxId, mut on_wound: F) -> AcquireDisposition<'_> where F: FnMut(GlobalTxId) -> Fut, - Fut: Future, + Fut: Future + Send + 'static, { let mut wounded_rx = match self.subscribe_wounded(&tx_id) { Some(rx) => rx, @@ -374,17 +396,44 @@ impl GlobalTxManager { } if let Some(owner) = owner_to_wound { + let wound_grace_period = self.wound_grace_period; log::info!( - "global transaction {tx_id} is waiting behind younger owner {owner}; triggering wound flow" + "global transaction {tx_id} is waiting behind younger owner {owner}; giving it {:?} to finish before wound flow", + wound_grace_period ); - if self.should_wound_locally(&owner) { - let _ = self.wound(&owner); - } else { + let owner_finished = tokio::select! { + changed = wounded_rx.changed(), if !*wounded_rx.borrow() => { + if changed.is_ok() && *wounded_rx.borrow() { + return AcquireDisposition::Cancelled; + } + false + } + _ = notify.notified() => true, + _ = tokio::time::sleep(wound_grace_period) => false, + }; + if owner_finished { + log::info!("global transaction {tx_id} observed owner {owner} finish within grace period; not triggering wound",); + continue; + } + + let should_trigger_wound = { + let state = self.lock_state.lock().unwrap(); + state.owner == Some(owner) + }; + if should_trigger_wound { log::info!( - "global transaction {tx_id} observed prepared participant owner {owner}; notifying coordinator without local wound" + "global transaction {tx_id} is still waiting behind younger owner {owner} after {:?}; triggering wound flow", + wound_grace_period ); + if self.should_wound_locally(&owner) { + let _ = self.wound(&owner); + } else { + log::info!( + "global transaction {tx_id} observed prepared participant owner {owner}; notifying coordinator without local wound" + ); + } + tokio::spawn(on_wound(owner)); } - on_wound(owner).await; } tokio::select! { @@ -498,6 +547,12 @@ mod tests { ) } + #[test] + fn manager_uses_configured_wound_grace_period() { + let manager = GlobalTxManager::new(Duration::from_millis(42)); + assert_eq!(manager.wound_grace_period(), Duration::from_millis(42)); + } + #[test] fn older_requester_wounds_younger_owner() { let manager = Arc::new(GlobalTxManager::default()); @@ -519,12 +574,41 @@ mod tests { AcquireDisposition::Cancelled => false, } }); - std::thread::sleep(Duration::from_millis(10)); + std::thread::sleep(Duration::from_millis(25)); assert!(manager.is_wounded(&younger)); drop(younger_guard); assert!(matches!(rt.block_on(older_task).expect("task should complete"), true)); } + #[test] + fn younger_owner_finishing_within_grace_period_is_not_wounded() { + let manager = Arc::new(GlobalTxManager::default()); + let younger = tx_id(20, 2, 0); + let older = tx_id(10, 1, 0); + manager.ensure_session(younger, super::GlobalTxRole::Participant, younger.creator_db); + manager.ensure_session(older, super::GlobalTxRole::Participant, older.creator_db); + + let rt = Runtime::new().unwrap(); + let younger_guard = match rt.block_on(manager.acquire(younger, |_| async {})) { + AcquireDisposition::Acquired(guard) => guard, + AcquireDisposition::Cancelled => panic!("younger tx should acquire immediately"), + }; + + let manager_for_task = manager.clone(); + let older_task = rt.spawn(async move { + match manager_for_task.acquire(older, |_| async {}).await { + AcquireDisposition::Acquired(_guard) => true, + AcquireDisposition::Cancelled => false, + } + }); + + std::thread::sleep(Duration::from_millis(5)); + drop(younger_guard); + + assert!(matches!(rt.block_on(older_task).expect("task should complete"), true)); + assert!(!manager.is_wounded(&younger)); + } + #[test] fn younger_requester_waits_behind_older_owner() { let manager = GlobalTxManager::default(); @@ -628,14 +712,17 @@ mod tests { }; let coordinator_wounded = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (wound_tx, wound_rx) = std::sync::mpsc::channel(); let flag = coordinator_wounded.clone(); let manager_for_task = manager.clone(); let older_task = rt.spawn(async move { match manager_for_task .acquire(older, move |_| { let flag = flag.clone(); + let wound_tx = wound_tx.clone(); async move { flag.store(true, Ordering::SeqCst); + let _ = wound_tx.send(()); } }) .await @@ -645,7 +732,9 @@ mod tests { } }); - std::thread::sleep(Duration::from_millis(10)); + wound_rx + .recv_timeout(Duration::from_millis(50)) + .expect("coordinator should be notified"); assert!(coordinator_wounded.load(Ordering::SeqCst)); assert!(!manager.is_wounded(&owner)); drop(owner_guard); diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index bca5278486b..fb32b3d035a 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -3,7 +3,7 @@ use super::scheduler::SchedulerStarter; use super::wasmtime::WasmtimeRuntime; use super::{Scheduler, UpdateDatabaseResult}; use crate::client::{ClientActorId, ClientName}; -use crate::config::V8HeapPolicyConfig; +use crate::config::{GlobalTxConfig, V8HeapPolicyConfig}; use crate::database_logger::DatabaseLogger; use crate::db::persistence::PersistenceProvider; use crate::db::relational_db::{self, spawn_view_cleanup_loop, DiskSizeFn, RelationalDB, Txdata}; @@ -138,6 +138,8 @@ pub struct HostController { /// /// `None` in test/embedded contexts where no JWT signer is configured. pub call_reducer_auth_token: Option, + /// Global distributed transaction tuning. + pub global_tx_config: GlobalTxConfig, } pub(crate) struct HostRuntimes { @@ -236,6 +238,7 @@ impl HostController { data_dir: Arc, default_config: db::Config, v8_heap_policy: V8HeapPolicyConfig, + global_tx_config: GlobalTxConfig, program_storage: ProgramStorage, energy_monitor: Arc, persistence: Arc, @@ -255,6 +258,7 @@ impl HostController { call_reducer_client: ReplicaContext::new_call_reducer_client(&CallReducerOnDbConfig::default()), call_reducer_router: Arc::new(LocalReducerRouter::new("http://127.0.0.1:3000")), call_reducer_auth_token: None, + global_tx_config, } } @@ -695,6 +699,7 @@ async fn make_replica_ctx( call_reducer_client: reqwest::Client, call_reducer_router: Arc, call_reducer_auth_token: Option, + global_tx_config: GlobalTxConfig, ) -> anyhow::Result { let logger = match module_logs { Some(path) => asyncify(move || Arc::new(DatabaseLogger::open_today(path))).await, @@ -731,7 +736,9 @@ async fn make_replica_ctx( call_reducer_router, call_reducer_auth_token, tx_id_nonce: Arc::default(), - global_tx_manager: Arc::default(), + global_tx_manager: Arc::new(crate::host::global_tx::GlobalTxManager::new( + global_tx_config.wound_grace_period, + )), }) } @@ -810,6 +817,7 @@ struct ModuleLauncher { call_reducer_client: reqwest::Client, call_reducer_router: Arc, call_reducer_auth_token: Option, + global_tx_config: GlobalTxConfig, } impl ModuleLauncher { @@ -832,6 +840,7 @@ impl ModuleLauncher { self.call_reducer_client, self.call_reducer_router, self.call_reducer_auth_token, + self.global_tx_config, ) .await .map(Arc::new)?; @@ -1036,6 +1045,7 @@ impl Host { call_reducer_client: host_controller.call_reducer_client.clone(), call_reducer_router: host_controller.call_reducer_router.clone(), call_reducer_auth_token: host_controller.call_reducer_auth_token.clone(), + global_tx_config: host_controller.global_tx_config, } .launch_module() .await? @@ -1068,6 +1078,7 @@ impl Host { call_reducer_client: host_controller.call_reducer_client.clone(), call_reducer_router: host_controller.call_reducer_router.clone(), call_reducer_auth_token: host_controller.call_reducer_auth_token.clone(), + global_tx_config: host_controller.global_tx_config, } .launch_module() .await; @@ -1094,6 +1105,7 @@ impl Host { call_reducer_client: host_controller.call_reducer_client.clone(), call_reducer_router: host_controller.call_reducer_router.clone(), call_reducer_auth_token: host_controller.call_reducer_auth_token.clone(), + global_tx_config: host_controller.global_tx_config, } .launch_module() .await; @@ -1210,6 +1222,7 @@ impl Host { call_reducer_client: ReplicaContext::new_call_reducer_client(&CallReducerOnDbConfig::default()), call_reducer_router: Arc::new(LocalReducerRouter::new("http://127.0.0.1:3000")), call_reducer_auth_token: None, + global_tx_config: GlobalTxConfig::default(), } .launch_module() .await diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 74b09be8933..a4e7889db1a 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2125,10 +2125,14 @@ impl ModuleHost { return Err(outcome); } + let this = self.clone(); match manager - .acquire(tx_id, |owner| async move { - if owner.creator_db != local_db { - self.send_wound_to_coordinator(owner).await; + .acquire(tx_id, move |owner| { + let this = this.clone(); + async move { + if owner.creator_db != local_db { + this.send_wound_to_coordinator(owner).await; + } } }) .await diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index e7d1aec30ea..b5cda9d27a1 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -34,4 +34,8 @@ directives = [ # Apply a V8 heap limit in MiB. Set to 0 to use V8's default limit. # heap-limit-mb = 0 +[global-tx] +# Delay before an older transaction wounds a younger owner. +# wound-grace-period = "10ms" + # vim: set nowritebackup: << otherwise triggers cargo-watch diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index d7264f409a3..4f676947eaa 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -10,7 +10,7 @@ use async_trait::async_trait; use clap::{ArgMatches, Command}; use http::StatusCode; use spacetimedb::client::ClientActorIndex; -use spacetimedb::config::{CertificateAuthority, MetadataFile, V8HeapPolicyConfig}; +use spacetimedb::config::{CertificateAuthority, GlobalTxConfig, MetadataFile, V8HeapPolicyConfig}; use spacetimedb::db; use spacetimedb::db::persistence::LocalPersistenceProvider; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; @@ -45,6 +45,7 @@ pub struct StandaloneOptions { pub db_config: db::Config, pub websocket: WebSocketOptions, pub v8_heap_policy: V8HeapPolicyConfig, + pub global_tx: GlobalTxConfig, /// HTTP base URL of this node's API server (e.g. `"http://127.0.0.1:3000"`). /// Used to configure the `LocalReducerRouter` so that cross-DB reducer calls /// reach the correct address when the server listens on a dynamic port. @@ -86,6 +87,7 @@ impl StandaloneEnv { data_dir, config.db_config, config.v8_heap_policy, + config.global_tx, program_store.clone(), energy_monitor, persistence_provider, @@ -658,6 +660,7 @@ mod tests { }, websocket: WebSocketOptions::default(), v8_heap_policy: V8HeapPolicyConfig::default(), + global_tx: GlobalTxConfig::default(), local_api_url: "http://127.0.0.1:3000".to_owned(), }; diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 4601da4ecd4..c31724f3c26 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -244,6 +244,7 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { db_config, websocket: config.websocket, v8_heap_policy: config.common.v8_heap_policy, + global_tx: config.common.global_tx, local_api_url: format!("http://127.0.0.1:{local_port}"), }, &certs, From 2dd26e13c04e04dc478b6e52026df24c91b98122 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Mon, 30 Mar 2026 22:56:11 -0700 Subject: [PATCH 10/33] Fix an error with the happy path --- crates/bindings-sys/src/lib.rs | 24 ++++++++++++++++-------- crates/bindings/src/remote_reducer.rs | 8 ++++++-- crates/guard/src/lib.rs | 6 ++++++ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 24b355983c0..56966cd84ab 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -1520,10 +1520,14 @@ pub fn call_reducer_on_db( // on transport failure. Unlike other ABI functions, a non-zero return value here // does NOT indicate a generic errno — it's the HTTP status code. Only HTTP_ERROR // specifically signals a transport-level failure. - match Errno::from_code(status) { - Some(errno @ (Errno::HTTP_ERROR | Errno::WOUNDED_TRANSACTION)) => Err((errno, out)), - Some(errno) => panic!("{errno:?}"), - None => Ok((status, out)), + if (100..=599).contains(&status) { + Ok((status, out)) + } else { + match Errno::from_code(status) { + Some(errno @ (Errno::HTTP_ERROR | Errno::WOUNDED_TRANSACTION)) => Err((errno, out)), + Some(errno) => panic!("{errno:?}"), + None => Ok((status, out)), + } } } @@ -1551,10 +1555,14 @@ pub fn call_reducer_on_db_2pc( &mut out, ) }; - match Errno::from_code(status) { - Some(errno @ (Errno::HTTP_ERROR | Errno::WOUNDED_TRANSACTION)) => Err((errno, out)), - Some(errno) => panic!("{errno:?}"), - None => Ok((status, out)), + if (100..=599).contains(&status) { + Ok((status, out)) + } else { + match Errno::from_code(status) { + Some(errno @ (Errno::HTTP_ERROR | Errno::WOUNDED_TRANSACTION)) => Err((errno, out)), + Some(errno) => panic!("{errno:?}"), + None => Ok((status, out)), + } } } diff --git a/crates/bindings/src/remote_reducer.rs b/crates/bindings/src/remote_reducer.rs index e57fea3633c..8ec58b6b786 100644 --- a/crates/bindings/src/remote_reducer.rs +++ b/crates/bindings/src/remote_reducer.rs @@ -78,7 +78,9 @@ pub fn call_reducer_on_db( Ok((status, body_source)) => { if status < 300 { let mut out = Vec::new(); - read_bytes_source_into(body_source, &mut out); + if body_source != spacetimedb_bindings_sys::raw::BytesSource::INVALID { + read_bytes_source_into(body_source, &mut out); + } return Ok(out); } // Decode the response body as the error message. @@ -125,7 +127,9 @@ pub fn call_reducer_on_db_2pc( Ok((status, body_source)) => { if status < 300 { let mut out = Vec::new(); - read_bytes_source_into(body_source, &mut out); + if body_source != spacetimedb_bindings_sys::raw::BytesSource::INVALID { + read_bytes_source_into(body_source, &mut out); + } return Ok(out); } let msg = if body_source == spacetimedb_bindings_sys::raw::BytesSource::INVALID { diff --git a/crates/guard/src/lib.rs b/crates/guard/src/lib.rs index af937910a54..cc96bfc87e9 100644 --- a/crates/guard/src/lib.rs +++ b/crates/guard/src/lib.rs @@ -92,6 +92,8 @@ pub fn ensure_binaries_built() -> PathBuf { use reqwest::blocking::Client; +const SMOKETEST_DEDICATED_DATABASE_CORES: &str = "1"; + pub struct SpacetimeDbGuard { pub child: Child, pub host_url: String, @@ -279,6 +281,10 @@ impl SpacetimeDbGuard { &data_dir_str, "--listen-addr", &address, + // Test-spawned servers should not inherit the CLI's machine-dependent + // default dedicated DB core count. + "--dedicated-database-cores", + SMOKETEST_DEDICATED_DATABASE_CORES, ]; if let Some(ref port) = pg_port_str { args.extend(["--pg-port", port]); From 9af221ff77afafcead9d44a8fbb775a25b6f42e2 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 31 Mar 2026 02:16:50 -0400 Subject: [PATCH 11/33] Changes to try to prevent deadlock --- crates/core/src/db/relational_db.rs | 1 + crates/core/src/host/mod.rs | 11 ++++++----- crates/core/src/host/module_host.rs | 13 ++++++++----- .../src/host/wasm_common/module_host_actor.rs | 18 +++++++++--------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 12837a34ad5..1949b7e571a 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -56,6 +56,7 @@ use spacetimedb_table::indexes::RowPointer; use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::table::{RowRef, TableScanIter}; use spacetimedb_table::table_index::IndexKey; +use tokio::task::spawn_blocking; use std::borrow::Cow; use std::io; use std::num::NonZeroUsize; diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 2213206138b..a1629f288f7 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -29,11 +29,12 @@ where F: Future + Send, F::Output: Send, { - std::thread::scope(|s| { - s.spawn(|| handle.block_on(fut)) - .join() - .expect("block_on_scoped: thread panicked") - }) + // std::thread::scope(|s| { + // s.spawn(|| handle.block_on(fut)) + // .join() + // .expect("block_on_scoped: thread panicked") + // }) + futures::executor::block_on(fut) } mod disk_storage; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index a4e7889db1a..97c2ed2fa27 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2237,11 +2237,14 @@ impl ModuleHost { /// Check whether `prepare_id` is present in the coordinator log of this database. /// Used by participant B to ask coordinator A: "did you commit?" - pub fn has_2pc_coordinator_commit(&self, prepare_id: &str) -> bool { - let db = self.relational_db(); - db.pending_2pc_coordinator_commits() - .map(|rows| rows.iter().any(|r| r.participant_prepare_id == prepare_id)) - .unwrap_or(false) + pub async fn has_2pc_coordinator_commit(&self, prepare_id: &str) -> bool { + let db = self.relational_db().clone(); + let prepare_id = prepare_id.to_string(); + tokio::task::spawn_blocking(move || { + db.pending_2pc_coordinator_commits() + .map(|rows| rows.iter().any(|r| r.participant_prepare_id == prepare_id)) + .unwrap_or(false) + }).await.expect("Couldn't spawn blocking task") } /// Crash recovery for the **coordinator** role. diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index b89373f4f96..079596cb5a6 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1269,15 +1269,15 @@ impl InstanceCommon { match req.send().await { Ok(resp) if resp.status().is_success() => { log::info!("2PC {action}: {prepare_id} on {db_identity}"); - if committed { - if let Err(e) = - stdb.with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |del_tx| { - Ok(del_tx.delete_st_2pc_coordinator_log(prepare_id)?) - }) - { - log::warn!("delete_st_2pc_coordinator_log failed for {prepare_id}: {e}"); - } - } + // if committed { + // if let Err(e) = + // stdb.with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |del_tx| { + // Ok(del_tx.delete_st_2pc_coordinator_log(prepare_id)?) + // }) + // { + // log::warn!("delete_st_2pc_coordinator_log failed for {prepare_id}: {e}"); + // } + // } } Ok(resp) => { log::error!( From e8069dfaca2d54cb05af8ac4be5694554211460d Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 31 Mar 2026 02:24:44 -0400 Subject: [PATCH 12/33] Fix typo --- crates/client-api/src/routes/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 189f00399fd..bfe699dcfe7 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -390,7 +390,7 @@ pub async fn status_2pc( ) -> axum::response::Result { let (module, _database) = find_module_and_database(&worker_ctx, name_or_identity).await?; - let decision = if module.has_2pc_coordinator_commit(&prepare_id) { + let decision = if module.has_2pc_coordinator_commit(&prepare_id).await { "commit" } else { "abort" From 11eb7ecac47b6d540703d232e1db5f8614e570e0 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 13:31:33 +0530 Subject: [PATCH 13/33] blocking req client --- Cargo.toml | 2 +- crates/core/src/client/client_connection.rs | 4 +- crates/core/src/db/relational_db.rs | 2 +- crates/core/src/host/host_controller.rs | 15 + crates/core/src/host/instance_env.rs | 276 +++++++----------- crates/core/src/host/mod.rs | 7 +- crates/core/src/host/module_host.rs | 13 +- crates/core/src/host/reducer_router.rs | 12 + .../src/host/wasm_common/module_host_actor.rs | 42 ++- .../src/host/wasmtime/wasm_instance_env.rs | 8 +- crates/core/src/replica_context.rs | 28 +- crates/core/src/startup.rs | 16 +- .../src/locking_tx_datastore/mut_tx.rs | 4 +- crates/primitives/src/errno.rs | 5 +- 14 files changed, 217 insertions(+), 217 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2e8daf523d7..87a2c459245 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -262,7 +262,7 @@ rand_distr = "0.5.1" rayon = "1.8" rayon-core = "1.11.0" regex = "1" -reqwest = { version = "0.12", features = ["stream", "json"] } +reqwest = { version = "0.12", features = ["stream", "json", "blocking"] } rolldown = { git = "https://github.com/rolldown/rolldown.git", tag = "v1.0.0-rc.3" } rolldown_common = { git = "https://github.com/rolldown/rolldown.git", tag = "v1.0.0-rc.3" } rolldown_error = { git = "https://github.com/rolldown/rolldown.git", tag = "v1.0.0-rc.3" } diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index efe7dfc2d0f..ebae782e949 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -23,7 +23,6 @@ use crate::util::prometheus_handle::IntGaugeExt; use crate::worker_metrics::WORKER_METRICS; use bytes::Bytes; use bytestring::ByteString; -use std::time::Duration; use derive_more::From; use futures::prelude::*; use prometheus::{Histogram, IntCounter, IntGauge}; @@ -33,6 +32,7 @@ use spacetimedb_durability::{DurableOffset, TxOffset}; use spacetimedb_lib::identity::{AuthCtx, RequestId}; use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::{bsatn, Identity, TimeDuration, Timestamp}; +use std::time::Duration; use tokio::sync::mpsc::error::{SendError, TrySendError}; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::AbortHandle; @@ -899,7 +899,7 @@ impl ClientConnection { } return Ok(result); } - + log::info!("Reducer call was wounded on attempt {attempt}, retrying after {wound_backoff:?} with new transaction ID {tx_id}"); sleep(wound_backoff).await; wound_backoff = wound_backoff.mul_f32(2.0).min(MAX_BACKOFF); diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 1949b7e571a..79c0d9593de 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -56,13 +56,13 @@ use spacetimedb_table::indexes::RowPointer; use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::table::{RowRef, TableScanIter}; use spacetimedb_table::table_index::IndexKey; -use tokio::task::spawn_blocking; use std::borrow::Cow; use std::io; use std::num::NonZeroUsize; use std::ops::RangeBounds; use std::sync::Arc; use tokio::sync::watch; +use tokio::task::spawn_blocking; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; pub use super::snapshot::SnapshotWorker; diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index fb32b3d035a..61dd71fb304 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -123,6 +123,8 @@ pub struct HostController { /// /// All per-replica clones share the same underlying connection pool. pub call_reducer_client: reqwest::Client, + /// Blocking variant of [`call_reducer_client`] for the WASM executor thread. + pub call_reducer_blocking_client: reqwest::blocking::Client, /// Router that resolves the HTTP base URL of the leader node for a given database. /// /// Set to [`LocalReducerRouter`] by default; replaced with `ClusterReducerRouter` @@ -256,6 +258,9 @@ impl HostController { bsatn_rlb_pool: BsatnRowListBuilderPool::new(), db_cores, call_reducer_client: ReplicaContext::new_call_reducer_client(&CallReducerOnDbConfig::default()), + call_reducer_blocking_client: ReplicaContext::new_call_reducer_blocking_client( + &CallReducerOnDbConfig::default(), + ), call_reducer_router: Arc::new(LocalReducerRouter::new("http://127.0.0.1:3000")), call_reducer_auth_token: None, global_tx_config, @@ -697,6 +702,7 @@ async fn make_replica_ctx( relational_db: Arc, bsatn_rlb_pool: BsatnRowListBuilderPool, call_reducer_client: reqwest::Client, + call_reducer_blocking_client: reqwest::blocking::Client, call_reducer_router: Arc, call_reducer_auth_token: Option, global_tx_config: GlobalTxConfig, @@ -733,6 +739,7 @@ async fn make_replica_ctx( logger, subscriptions, call_reducer_client, + call_reducer_blocking_client, call_reducer_router, call_reducer_auth_token, tx_id_nonce: Arc::default(), @@ -815,6 +822,7 @@ struct ModuleLauncher { core: AllocatedJobCore, bsatn_rlb_pool: BsatnRowListBuilderPool, call_reducer_client: reqwest::Client, + call_reducer_blocking_client: reqwest::blocking::Client, call_reducer_router: Arc, call_reducer_auth_token: Option, global_tx_config: GlobalTxConfig, @@ -838,6 +846,7 @@ impl ModuleLauncher { self.relational_db, self.bsatn_rlb_pool, self.call_reducer_client, + self.call_reducer_blocking_client, self.call_reducer_router, self.call_reducer_auth_token, self.global_tx_config, @@ -1043,6 +1052,7 @@ impl Host { core: host_controller.db_cores.take(), bsatn_rlb_pool: bsatn_rlb_pool.clone(), call_reducer_client: host_controller.call_reducer_client.clone(), + call_reducer_blocking_client: host_controller.call_reducer_blocking_client.clone(), call_reducer_router: host_controller.call_reducer_router.clone(), call_reducer_auth_token: host_controller.call_reducer_auth_token.clone(), global_tx_config: host_controller.global_tx_config, @@ -1076,6 +1086,7 @@ impl Host { core: host_controller.db_cores.take(), bsatn_rlb_pool: bsatn_rlb_pool.clone(), call_reducer_client: host_controller.call_reducer_client.clone(), + call_reducer_blocking_client: host_controller.call_reducer_blocking_client.clone(), call_reducer_router: host_controller.call_reducer_router.clone(), call_reducer_auth_token: host_controller.call_reducer_auth_token.clone(), global_tx_config: host_controller.global_tx_config, @@ -1103,6 +1114,7 @@ impl Host { core: host_controller.db_cores.take(), bsatn_rlb_pool: bsatn_rlb_pool.clone(), call_reducer_client: host_controller.call_reducer_client.clone(), + call_reducer_blocking_client: host_controller.call_reducer_blocking_client.clone(), call_reducer_router: host_controller.call_reducer_router.clone(), call_reducer_auth_token: host_controller.call_reducer_auth_token.clone(), global_tx_config: host_controller.global_tx_config, @@ -1220,6 +1232,9 @@ impl Host { bsatn_rlb_pool, // Transient validation-only module; build its own client and router with defaults. call_reducer_client: ReplicaContext::new_call_reducer_client(&CallReducerOnDbConfig::default()), + call_reducer_blocking_client: ReplicaContext::new_call_reducer_blocking_client( + &CallReducerOnDbConfig::default(), + ), call_reducer_router: Arc::new(LocalReducerRouter::new("http://127.0.0.1:3000")), call_reducer_auth_token: None, global_tx_config: GlobalTxConfig::default(), diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 744417acead..67fa3630d21 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1026,9 +1026,8 @@ impl InstanceEnv { /// Unlike [`Self::http_request`], this is explicitly allowed while a transaction is open — /// the caller is responsible for understanding the consistency implications. /// - /// Uses [`ReplicaContext::call_reducer_router`] to resolve the leader node for - /// `database_identity`, then sends the request via the warmed HTTP client in - /// [`ReplicaContext::call_reducer_client`]. + /// Blocks the calling thread (the `SingleCoreExecutor` OS thread) for the duration of the + /// HTTP round-trip using `reqwest::blocking::Client`. No tokio involvement on this path. /// /// Returns `(http_status, response_body)` on transport success, /// or [`NodesError::HttpError`] if the connection itself fails. @@ -1037,72 +1036,59 @@ impl InstanceEnv { database_identity: Identity, reducer_name: &str, args: bytes::Bytes, - ) -> impl Future> + use<> { - let client = self.replica_ctx.call_reducer_client.clone(); - let router = self.replica_ctx.call_reducer_router.clone(); - let reducer_name = reducer_name.to_owned(); - // Node-level auth token: a single token minted at startup and shared by all replicas - // on this node. Passed as a Bearer token so `anon_auth_middleware` on the target node - // accepts the request without generating a fresh ephemeral identity per call. - let auth_token = self.replica_ctx.call_reducer_auth_token.clone(); - let caller_identity = self.replica_ctx.database.database_identity; + ) -> Result<(u16, bytes::Bytes), NodesError> { let tx_id = self.current_tx_id(); - let wounded_error = tx_id.and_then(|tx_id| { - self.replica_ctx - .global_tx_manager - .is_wounded(&tx_id) - .then(|| self.wounded_tx_error(tx_id)) - }); - - async move { - if let Some(err) = wounded_error { - return Err(err); + if let Some(tx_id) = tx_id { + if self.replica_ctx.global_tx_manager.is_wounded(&tx_id) { + return Err(self.wounded_tx_error(tx_id)); } + } - let start = Instant::now(); + let start = Instant::now(); + let caller_identity = self.replica_ctx.database.database_identity; - let base_url = router - .resolve_base_url(database_identity) - .await - .map_err(|e| NodesError::HttpError(e.to_string()))?; - let url = format!( - "{}/v1/database/{}/call/{}", - base_url, - database_identity.to_hex(), - reducer_name, - ); - let mut req = client - .post(&url) - .header(http::header::CONTENT_TYPE, "application/octet-stream") - .body(args); - if let Some(token) = auth_token { - req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); - } - if let Some(tx_id) = tx_id { - req = req.header(TX_ID_HEADER, tx_id.to_string()); - } - let result = async { - let response = req.send().await.map_err(|e| NodesError::HttpError(e.to_string()))?; - let status = response.status().as_u16(); - let body = response - .bytes() - .await - .map_err(|e| NodesError::HttpError(e.to_string()))?; - Ok::<_, NodesError>((status, body)) - } - .await; - - WORKER_METRICS - .cross_db_reducer_calls_total - .with_label_values(&caller_identity) - .inc(); - WORKER_METRICS - .cross_db_reducer_duration_seconds - .with_label_values(&caller_identity) - .observe(start.elapsed().as_secs_f64()); - - result + let base_url = self + .replica_ctx + .call_reducer_router + .resolve_base_url_blocking(database_identity) + .map_err(|e| NodesError::HttpError(e.to_string()))?; + let url = format!( + "{}/v1/database/{}/call/{}", + base_url, + database_identity.to_hex(), + reducer_name, + ); + let mut req = self + .replica_ctx + .call_reducer_blocking_client + .post(&url) + .header(http::header::CONTENT_TYPE, "application/octet-stream") + .body(args.to_vec()); + if let Some(ref token) = self.replica_ctx.call_reducer_auth_token { + req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + } + if let Some(tx_id) = tx_id { + req = req.header(TX_ID_HEADER, tx_id.to_string()); } + let result = req + .send() + .map_err(|e| NodesError::HttpError(e.to_string())) + .and_then(|resp| { + let status = resp.status().as_u16(); + let body = resp.bytes().map_err(|e| NodesError::HttpError(e.to_string()))?; + Ok((status, body)) + }); + + WORKER_METRICS + .cross_db_reducer_calls_total + .with_label_values(&caller_identity) + .inc(); + WORKER_METRICS + .cross_db_reducer_duration_seconds + .with_label_values(&caller_identity) + .observe(start.elapsed().as_secs_f64()); + + result } /// Call a reducer on a remote database using the 2PC prepare protocol. @@ -1111,123 +1097,84 @@ impl InstanceEnv { /// `/call/{reducer}`. On success, parses the `X-Prepare-Id` response header and stores /// `(database_identity, prepare_id)` in [`Self::prepared_participants`]. /// - /// Returns `(http_status, response_body)` on transport success. - /// The caller (coordinator reducer) is responsible for checking the status; - /// if the coordinator's reducer commits, the runtime will commit all participants, - /// and if it fails, the runtime will abort them. + /// Blocks the calling thread for the duration of the HTTP round-trip. + /// + /// Returns `(http_status, response_body, prepare_id)` on transport success. pub fn call_reducer_on_db_2pc( &mut self, database_identity: Identity, reducer_name: &str, args: bytes::Bytes, - ) -> impl Future), NodesError>> + use<> { - let client = self.replica_ctx.call_reducer_client.clone(); - let router = self.replica_ctx.call_reducer_router.clone(); - let reducer_name = reducer_name.to_owned(); - let auth_token = self.replica_ctx.call_reducer_auth_token.clone(); + ) -> Result<(u16, bytes::Bytes, Option), NodesError> { let caller_identity = self.replica_ctx.database.database_identity; - let tx_id = self.current_tx_id(); - let wounded_error = tx_id.and_then(|tx_id| { - self.replica_ctx - .global_tx_manager - .is_wounded(&tx_id) - .then(|| self.wounded_tx_error(tx_id)) - }); - let wounded_rx = tx_id.and_then(|tx_id| self.replica_ctx.global_tx_manager.subscribe_wounded(&tx_id)); - let wounded_message = - tx_id.map(|tx_id| format!("distributed transaction {tx_id} was wounded by an older transaction")); + let tx_id = self.current_tx_id().ok_or_else(|| { + NodesError::HttpError("2PC remote reducer call requires an active distributed transaction id".to_owned()) + })?; - if let Some(tx_id) = tx_id { - let session = - self.replica_ctx - .global_tx_manager - .ensure_session(tx_id, GlobalTxRole::Coordinator, tx_id.creator_db); - session.set_state(GlobalTxState::Preparing); + if self.replica_ctx.global_tx_manager.is_wounded(&tx_id) { + return Err(self.wounded_tx_error(tx_id)); } - async move { - let tx_id = tx_id.ok_or_else(|| { - NodesError::HttpError( - "2PC remote reducer call requires an active distributed transaction id".to_owned(), - ) - })?; - if let Some(err) = wounded_error { - return Err(err); - } + let session = + self.replica_ctx + .global_tx_manager + .ensure_session(tx_id, GlobalTxRole::Coordinator, tx_id.creator_db); + session.set_state(GlobalTxState::Preparing); + + let start = Instant::now(); + + let base_url = self + .replica_ctx + .call_reducer_router + .resolve_base_url_blocking(database_identity) + .map_err(|e| NodesError::HttpError(e.to_string()))?; + let url = format!( + "{}/v1/database/{}/prepare/{}", + base_url, + database_identity.to_hex(), + reducer_name, + ); + let mut req = self + .replica_ctx + .call_reducer_blocking_client + .post(&url) + .header(http::header::CONTENT_TYPE, "application/octet-stream") + .header("X-Coordinator-Identity", caller_identity.to_hex().to_string()) + .header(TX_ID_HEADER, tx_id.to_string()) + .body(args.to_vec()); + if let Some(ref token) = self.replica_ctx.call_reducer_auth_token { + req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + } - let start = Instant::now(); + // Check for wound signal one more time before blocking. + if self.replica_ctx.global_tx_manager.is_wounded(&tx_id) { + return Err(self.wounded_tx_error(tx_id)); + } - let base_url = router - .resolve_base_url(database_identity) - .await - .map_err(|e| NodesError::HttpError(e.to_string()))?; - let url = format!( - "{}/v1/database/{}/prepare/{}", - base_url, - database_identity.to_hex(), - reducer_name, - ); - let mut req = client - .post(&url) - .header(http::header::CONTENT_TYPE, "application/octet-stream") - .header("X-Coordinator-Identity", caller_identity.to_hex().to_string()) - .body(args); - if let Some(token) = auth_token { - req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); - } - req = req.header(TX_ID_HEADER, tx_id.to_string()); - let request_fut = async { - let response = req.send().await.map_err(|e| NodesError::HttpError(e.to_string()))?; - let status = response.status().as_u16(); - let prepare_id = response + let result = req + .send() + .map_err(|e| NodesError::HttpError(e.to_string())) + .and_then(|resp| { + let status = resp.status().as_u16(); + let prepare_id = resp .headers() .get("X-Prepare-Id") .and_then(|v| v.to_str().ok()) .map(|s| s.to_owned()); - let body = response - .bytes() - .await - .map_err(|e| NodesError::HttpError(e.to_string()))?; + let body = resp.bytes().map_err(|e| NodesError::HttpError(e.to_string()))?; Ok((status, body, prepare_id)) - }; - tokio::pin!(request_fut); - let result = if let Some(mut wounded_rx) = wounded_rx { - tokio::select! { - result = &mut request_fut => result, - changed = wounded_rx.changed() => { - match changed { - Ok(()) if *wounded_rx.borrow() => { - log::info!( - "transaction {tx_id} was wounded during remote reducer call to {}/{}; aborting call", - database_identity.to_hex(), - reducer_name - ); - Err(NodesError::Wounded( - wounded_message - .clone() - .expect("wounded message should exist when tx_id exists"), - )) - }, - Ok(()) => request_fut.as_mut().await, - Err(_) => request_fut.as_mut().await, - } - } - } - } else { - request_fut.as_mut().await - }; + }); - WORKER_METRICS - .cross_db_reducer_calls_total - .with_label_values(&caller_identity) - .inc(); - WORKER_METRICS - .cross_db_reducer_duration_seconds - .with_label_values(&caller_identity) - .observe(start.elapsed().as_secs_f64()); + WORKER_METRICS + .cross_db_reducer_calls_total + .with_label_values(&caller_identity) + .inc(); + WORKER_METRICS + .cross_db_reducer_duration_seconds + .with_label_values(&caller_identity) + .observe(start.elapsed().as_secs_f64()); - result - } + result } } @@ -1604,6 +1551,9 @@ mod test { logger, subscriptions: subs, call_reducer_client: ReplicaContext::new_call_reducer_client(&CallReducerOnDbConfig::default()), + call_reducer_blocking_client: ReplicaContext::new_call_reducer_blocking_client( + &CallReducerOnDbConfig::default(), + ), call_reducer_router: Arc::new(LocalReducerRouter::new("http://127.0.0.1:3000")), call_reducer_auth_token: None, tx_id_nonce: Arc::default(), diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index a1629f288f7..e673831dc4f 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -24,16 +24,11 @@ use spacetimedb_schema::def::ModuleDef; /// /// Use this for every place in the 2PC / cross-DB call paths that needs to /// synchronously drive a future from blocking (WASM executor) context. -pub(crate) fn block_on_scoped(handle: &tokio::runtime::Handle, fut: F) -> F::Output +pub(crate) fn block_on_scoped(_handle: &tokio::runtime::Handle, fut: F) -> F::Output where F: Future + Send, F::Output: Send, { - // std::thread::scope(|s| { - // s.spawn(|| handle.block_on(fut)) - // .join() - // .expect("block_on_scoped: thread panicked") - // }) futures::executor::block_on(fut) } diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 97c2ed2fa27..11d129acb96 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2244,7 +2244,9 @@ impl ModuleHost { db.pending_2pc_coordinator_commits() .map(|rows| rows.iter().any(|r| r.participant_prepare_id == prepare_id)) .unwrap_or(false) - }).await.expect("Couldn't spawn blocking task") + }) + .await + .expect("Couldn't spawn blocking task") } /// Crash recovery for the **coordinator** role. @@ -2397,7 +2399,14 @@ impl ModuleHost { // Step 1: Re-run the reducer to reacquire the write lock. let new_prepare_id = match this - .prepare_reducer(caller_identity, Some(caller_connection_id), recovered_tx_id, &row.reducer_name, args, Some(coordinator_identity)) + .prepare_reducer( + caller_identity, + Some(caller_connection_id), + recovered_tx_id, + &row.reducer_name, + args, + Some(coordinator_identity), + ) .await { Ok((pid, result, _rv)) if !pid.is_empty() => { diff --git a/crates/core/src/host/reducer_router.rs b/crates/core/src/host/reducer_router.rs index dcbf20c51c8..c95de41a2b0 100644 --- a/crates/core/src/host/reducer_router.rs +++ b/crates/core/src/host/reducer_router.rs @@ -28,6 +28,14 @@ pub trait ReducerCallRouter: Send + Sync + 'static { /// Returns an error string when the leader cannot be resolved /// (database not found, no leader elected yet, node has no network address, etc.). fn resolve_base_url<'a>(&'a self, database_identity: Identity) -> BoxFuture<'a, anyhow::Result>; + + /// Blocking variant of [`resolve_base_url`] for use on non-async threads. + /// + /// The default implementation drives the async version via the current tokio handle. + /// Override for routers that can resolve without async (e.g. [`LocalReducerRouter`]). + fn resolve_base_url_blocking(&self, database_identity: Identity) -> anyhow::Result { + tokio::runtime::Handle::current().block_on(self.resolve_base_url(database_identity)) + } } // Arc is itself a ReducerCallRouter. @@ -60,4 +68,8 @@ impl ReducerCallRouter for LocalReducerRouter { let url = self.base_url.clone(); Box::pin(async move { Ok(url) }) } + + fn resolve_base_url_blocking(&self, _database_identity: Identity) -> anyhow::Result { + Ok(self.base_url.clone()) + } } diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 079596cb5a6..c9d2a6a5705 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -726,10 +726,11 @@ impl WasmModuleInstance { // B must not claim PREPARED until the marker is on disk — if B crashes after // claiming PREPARED but before the marker is durable, recovery has nothing to recover. if let Some(prepare_offset) = marker_tx_data.tx_offset() - && let Some(mut durable) = stdb.durable_tx_offset() { - let handle = tokio::runtime::Handle::current(); - let _ = block_on_scoped(&handle, durable.wait_for(prepare_offset)); - } + && let Some(mut durable) = stdb.durable_tx_offset() + { + let handle = tokio::runtime::Handle::current(); + let _ = block_on_scoped(&handle, durable.wait_for(prepare_offset)); + } // Step 4: signal PREPARED. let res = ReducerCallResult { @@ -841,21 +842,17 @@ impl WasmModuleInstance { } } - let handle = tokio::runtime::Handle::current(); - let client = replica_ctx.call_reducer_client.clone(); + let client = replica_ctx.call_reducer_blocking_client.clone(); let router = replica_ctx.call_reducer_router.clone(); let auth_token = replica_ctx.call_reducer_auth_token.clone(); let prepare_id_owned = prepare_id.to_owned(); loop { - let decision = block_on_scoped( - &handle, - Self::query_coordinator_status( - &client, - &router, - auth_token.clone(), - coordinator_identity, - &prepare_id_owned, - ), + let decision = Self::query_coordinator_status( + &client, + &router, + auth_token.clone(), + coordinator_identity, + &prepare_id_owned, ); match decision { Some(commit) => return commit, @@ -866,15 +863,16 @@ impl WasmModuleInstance { /// Query `GET /v1/database/{coordinator}/2pc/status/{prepare_id}`. /// - /// Returns `Some(true)` = COMMIT, `Some(false)` = ABORT, `None` = transient error (retry). - async fn query_coordinator_status( - client: &reqwest::Client, + /// Blocks the calling thread. Returns `Some(true)` = COMMIT, `Some(false)` = ABORT, + /// `None` = transient error (retry). + fn query_coordinator_status( + client: &reqwest::blocking::Client, router: &std::sync::Arc, auth_token: Option, coordinator_identity: crate::identity::Identity, prepare_id: &str, ) -> Option { - let base_url = match router.resolve_base_url(coordinator_identity).await { + let base_url = match router.resolve_base_url_blocking(coordinator_identity) { Ok(url) => url, Err(e) => { log::warn!("2PC status poll: cannot resolve coordinator URL: {e}"); @@ -888,12 +886,12 @@ impl WasmModuleInstance { prepare_id, ); let mut req = client.get(&url); - if let Some(token) = &auth_token { + if let Some(ref token) = auth_token { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } - match req.send().await { + match req.send() { Ok(resp) if resp.status().is_success() => { - let body = resp.text().await.unwrap_or_default(); + let body = resp.text().unwrap_or_default(); Some(body.trim() == "commit") } Ok(resp) => { diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 638b2777d13..721db4215c0 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -1997,11 +1997,9 @@ impl WasmInstanceEnv { let args_buf = mem.deref_slice(args_ptr, args_len)?; let args = bytes::Bytes::copy_from_slice(args_buf); - let handle = tokio::runtime::Handle::current(); - let fut = env + let result = env .instance_env .call_reducer_on_db(database_identity, &reducer_name, args); - let result = super::super::block_on_scoped(&handle, fut); match result { Ok((status, body)) => { @@ -2064,11 +2062,9 @@ impl WasmInstanceEnv { let args_buf = mem.deref_slice(args_ptr, args_len)?; let args = bytes::Bytes::copy_from_slice(args_buf); - let handle = tokio::runtime::Handle::current(); - let fut = env + let result = env .instance_env .call_reducer_on_db_2pc(database_identity, &reducer_name, args); - let result = super::super::block_on_scoped(&handle, fut); match result { Ok((status, body, prepare_id)) => { diff --git a/crates/core/src/replica_context.rs b/crates/core/src/replica_context.rs index 99d1dd0ba36..fd342925b3a 100644 --- a/crates/core/src/replica_context.rs +++ b/crates/core/src/replica_context.rs @@ -50,10 +50,19 @@ pub struct ReplicaContext { pub replica_id: u64, pub logger: Arc, pub subscriptions: ModuleSubscriptions, - /// Warmed HTTP/2 client for [`crate::host::instance_env::InstanceEnv::call_reducer_on_db`]. + /// Async HTTP/2 client for fire-and-forget coordinator/recovery tasks that run inside + /// tokio async tasks (e.g. `recover_2pc_coordinator`, `send_ack_commit_to_coordinator`). /// /// `reqwest::Client` is internally an `Arc`, so cloning `ReplicaContext` shares the pool. pub call_reducer_client: reqwest::Client, + /// Blocking HTTP client for cross-db calls made directly from the WASM executor thread. + /// + /// Used by [`crate::host::instance_env::InstanceEnv::call_reducer_on_db`] and the + /// 2PC participant's `wait_for_2pc_decision` polling loop, both of which run on the + /// `SingleCoreExecutor` std::thread and must block without yielding to tokio. + /// + /// `reqwest::blocking::Client` is also internally an `Arc`. + pub call_reducer_blocking_client: reqwest::blocking::Client, /// Resolves the HTTP base URL of the leader node for a given database identity. /// /// - Standalone: always returns the local node URL ([`crate::host::reducer_router::LocalReducerRouter`]). @@ -74,7 +83,7 @@ pub struct ReplicaContext { } impl ReplicaContext { - /// Build a warmed `reqwest::Client` from `config`. + /// Build a warmed async `reqwest::Client` from `config`. /// /// Uses HTTP/2 prior knowledge (h2c) for all connections. /// The server must be configured to accept h2c (HTTP/2 cleartext) connections. @@ -89,6 +98,21 @@ impl ReplicaContext { .expect("failed to build call_reducer_on_db HTTP client") } + /// Build a warmed blocking `reqwest::blocking::Client` from `config`. + /// + /// Used by the WASM executor thread and 2PC participant polling loop, which block their + /// OS thread synchronously rather than yielding to tokio. + pub fn new_call_reducer_blocking_client(config: &CallReducerOnDbConfig) -> reqwest::blocking::Client { + reqwest::blocking::Client::builder() + .tcp_keepalive(config.tcp_keepalive) + .pool_idle_timeout(config.pool_idle_timeout) + .pool_max_idle_per_host(config.pool_max_idle_per_host) + .timeout(config.request_timeout) + .http2_prior_knowledge() + .build() + .expect("failed to build call_reducer_on_db blocking HTTP client") + } + pub fn mint_global_tx_id(&self, start_ts: Timestamp) -> GlobalTxId { let nonce = self.tx_id_nonce.fetch_add(1, Ordering::Relaxed); GlobalTxId::new(start_ts, self.database.database_identity, nonce, 0) diff --git a/crates/core/src/startup.rs b/crates/core/src/startup.rs index 79694237edd..a0eeda9da77 100644 --- a/crates/core/src/startup.rs +++ b/crates/core/src/startup.rs @@ -413,10 +413,13 @@ impl Cores { #[cfg(target_os = "linux")] fn cores_to_cpuset(cores: &[CoreId]) -> Option { - cores.iter().copied().try_fold(nix::sched::CpuSet::new(), |mut cpuset, core| { - cpuset.set(core.id).ok()?; - Some(cpuset) - }) + cores + .iter() + .copied() + .try_fold(nix::sched::CpuSet::new(), |mut cpuset, core| { + cpuset.set(core.id).ok()?; + Some(cpuset) + }) } #[cfg(target_os = "linux")] @@ -591,10 +594,7 @@ mod tests { #[cfg(target_os = "linux")] { assert!(split.tokio.workers.is_none()); - assert_eq!( - cpuset_cardinality(split.tokio.blocking.as_ref().unwrap()), - 20 - ); + assert_eq!(cpuset_cardinality(split.tokio.blocking.as_ref().unwrap()), 20); assert!(split.rayon.dedicated.is_none()); assert_eq!(split.rayon.shared.as_ref().unwrap().0, 20); } diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index bd1385a04c9..dbdf5a1427e 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -2669,9 +2669,7 @@ impl MutTxId { /// Return all rows in `st_2pc_state` (prepared but not yet committed/aborted). /// Used on recovery: each row describes a transaction to resume. pub fn scan_st_2pc_state(&self) -> Result> { - self.iter(ST_2PC_STATE_ID)? - .map(St2pcStateRow::try_from) - .collect() + self.iter(ST_2PC_STATE_ID)?.map(St2pcStateRow::try_from).collect() } /// Insert a row into `st_2pc_coordinator_log` recording that the coordinator has diff --git a/crates/primitives/src/errno.rs b/crates/primitives/src/errno.rs index 8221e5a447c..7bd79c4d3a8 100644 --- a/crates/primitives/src/errno.rs +++ b/crates/primitives/src/errno.rs @@ -35,7 +35,10 @@ macro_rules! errnos { "ABI call can only be made while within a read-only transaction" ), HTTP_ERROR(21, "The HTTP request failed"), - WOUNDED_TRANSACTION(22, "The distributed transaction was wounded by an older transaction"), + WOUNDED_TRANSACTION( + 22, + "The distributed transaction was wounded by an older transaction" + ), ); }; } From 33d86b1306dee91e8ee1475c5bb495cbaac4c267 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 13:58:15 +0530 Subject: [PATCH 14/33] future executor --- crates/core/src/host/reducer_router.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/core/src/host/reducer_router.rs b/crates/core/src/host/reducer_router.rs index c95de41a2b0..815ad6ce876 100644 --- a/crates/core/src/host/reducer_router.rs +++ b/crates/core/src/host/reducer_router.rs @@ -31,10 +31,14 @@ pub trait ReducerCallRouter: Send + Sync + 'static { /// Blocking variant of [`resolve_base_url`] for use on non-async threads. /// - /// The default implementation drives the async version via the current tokio handle. - /// Override for routers that can resolve without async (e.g. [`LocalReducerRouter`]). + /// The default implementation drives the async version on a fresh OS thread with its own + /// minimal tokio runtime, so it is safe to call from any thread — including threads that + /// are already inside a tokio `block_on` context (e.g. the `SingleCoreExecutor` thread). + /// + /// Override for routers that can resolve without spawning (e.g. [`LocalReducerRouter`]). fn resolve_base_url_blocking(&self, database_identity: Identity) -> anyhow::Result { - tokio::runtime::Handle::current().block_on(self.resolve_base_url(database_identity)) + let fut = self.resolve_base_url(database_identity); + futures::executor::block_on(fut) } } From 199075c4768396c974b51474d76acbb5ce528855 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 13:58:56 +0530 Subject: [PATCH 15/33] full sync --- crates/core/src/host/instance_env.rs | 32 ++-- crates/core/src/host/module_host.rs | 1 - .../src/host/wasm_common/module_host_actor.rs | 168 +++++++++--------- crates/core/src/replica_context.rs | 62 ++++++- 4 files changed, 159 insertions(+), 104 deletions(-) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 67fa3630d21..1265ac2548f 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -5,7 +5,7 @@ use crate::error::{DBError, DatastoreError, IndexError, NodesError}; use crate::host::global_tx::{GlobalTxRole, GlobalTxState}; use crate::host::module_host::{DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall}; use crate::host::wasm_common::TimingSpan; -use crate::replica_context::ReplicaContext; +use crate::replica_context::{execute_blocking_http, ReplicaContext}; use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, ModuleSubscriptions}; use crate::subscription::module_subscription_manager::{from_tx_offset, TransactionOffset}; use crate::util::prometheus_handle::IntGaugeExt; @@ -1070,14 +1070,17 @@ impl InstanceEnv { if let Some(tx_id) = tx_id { req = req.header(TX_ID_HEADER, tx_id.to_string()); } - let result = req - .send() - .map_err(|e| NodesError::HttpError(e.to_string())) - .and_then(|resp| { + let request = req.build().map_err(|e| NodesError::HttpError(e.to_string()))?; + let result = execute_blocking_http( + &self.replica_ctx.call_reducer_blocking_client, + request, + |resp| { let status = resp.status().as_u16(); - let body = resp.bytes().map_err(|e| NodesError::HttpError(e.to_string()))?; + let body = resp.bytes()?; Ok((status, body)) - }); + }, + ) + .map_err(|e| NodesError::HttpError(e.to_string())); WORKER_METRICS .cross_db_reducer_calls_total @@ -1151,19 +1154,22 @@ impl InstanceEnv { return Err(self.wounded_tx_error(tx_id)); } - let result = req - .send() - .map_err(|e| NodesError::HttpError(e.to_string())) - .and_then(|resp| { + let request = req.build().map_err(|e| NodesError::HttpError(e.to_string()))?; + let result = execute_blocking_http( + &self.replica_ctx.call_reducer_blocking_client, + request, + |resp| { let status = resp.status().as_u16(); let prepare_id = resp .headers() .get("X-Prepare-Id") .and_then(|v| v.to_str().ok()) .map(|s| s.to_owned()); - let body = resp.bytes().map_err(|e| NodesError::HttpError(e.to_string()))?; + let body = resp.bytes()?; Ok((status, body, prepare_id)) - }); + }, + ) + .map_err(|e| NodesError::HttpError(e.to_string())); WORKER_METRICS .cross_db_reducer_calls_total diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 11d129acb96..3819fa09022 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1892,7 +1892,6 @@ impl ModuleHost { let prepare_tx_component = tx_id .map(|tx_id| tx_id.to_string()) .unwrap_or_else(|| format!("legacy:{}:00000000", caller_identity.to_hex())); - let coordinator_hex = coordinator_identity.to_hex(); let prepare_id = format!( "prepare-{}-{}", prepare_tx_component, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index c9d2a6a5705..c86db795ae2 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -21,7 +21,7 @@ use crate::host::{ use crate::identity::Identity; use crate::messages::control_db::HostType; use crate::module_host_context::ModuleCreationContext; -use crate::replica_context::ReplicaContext; +use crate::replica_context::{execute_blocking_http, ReplicaContext}; use crate::sql::ast::SchemaViewer; use crate::sql::execute::run_with_instance; use crate::subscription::module_subscription_actor::commit_and_broadcast_event; @@ -405,14 +405,14 @@ impl WasmModuleHostActor { /// /// Called AFTER B's commit is durable. Fire-and-forget: failure is tolerated because /// `recover_2pc_coordinator` on A will retransmit COMMIT on restart. -async fn send_ack_commit_to_coordinator( - client: reqwest::Client, - router: std::sync::Arc, - auth_token: Option, +fn send_ack_commit_to_coordinator( + client: &reqwest::blocking::Client, + router: &std::sync::Arc, + auth_token: &Option, coordinator_identity: crate::identity::Identity, - prepare_id: String, + prepare_id: &str, ) { - let base_url = match router.resolve_base_url(coordinator_identity).await { + let base_url = match router.resolve_base_url_blocking(coordinator_identity) { Ok(url) => url, Err(e) => { log::warn!("2PC ack-commit: cannot resolve coordinator URL: {e}"); @@ -426,18 +426,16 @@ async fn send_ack_commit_to_coordinator( prepare_id, ); let mut req = client.post(&url); - if let Some(token) = &auth_token { + if let Some(token) = auth_token { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } - match req.send().await { - Ok(resp) if resp.status().is_success() => { + let Ok(request) = req.build() else { return }; + match execute_blocking_http(client, request, |resp| Ok(resp.status())) { + Ok(status) if status.is_success() => { log::info!("2PC ack-commit: notified coordinator for {prepare_id}"); } - Ok(resp) => { - log::warn!( - "2PC ack-commit: coordinator returned {} for {prepare_id}", - resp.status() - ); + Ok(status) => { + log::warn!("2PC ack-commit: coordinator returned {status} for {prepare_id}"); } Err(e) => { log::warn!("2PC ack-commit: transport error for {prepare_id}: {e}"); @@ -774,18 +772,14 @@ impl WasmModuleInstance { } // Notify coordinator that B has committed so it can delete its coordinator log entry. - // Fire-and-forget: if this fails, coordinator's recover_2pc_coordinator will retry on - // restart, and B's commit_prepared will then return a harmless "not found" error. - let router = replica_ctx.call_reducer_router.clone(); - let client_http = replica_ctx.call_reducer_client.clone(); - let auth_token = replica_ctx.call_reducer_auth_token.clone(); - tokio::runtime::Handle::current().spawn(send_ack_commit_to_coordinator( - client_http, - router, - auth_token, + // Best-effort: if this fails, coordinator's recover_2pc_coordinator will retry on restart. + send_ack_commit_to_coordinator( + &replica_ctx.call_reducer_blocking_client, + &replica_ctx.call_reducer_router, + &replica_ctx.call_reducer_auth_token, coordinator_identity, - prepare_id, - )); + &prepare_id, + ); if let Some(tx_id) = global_tx_id { replica_ctx .global_tx_manager @@ -889,13 +883,21 @@ impl WasmModuleInstance { if let Some(ref token) = auth_token { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } - match req.send() { - Ok(resp) if resp.status().is_success() => { - let body = resp.text().unwrap_or_default(); - Some(body.trim() == "commit") + let request = match req.build() { + Ok(r) => r, + Err(e) => { + log::warn!("2PC status poll: failed to build request: {e}"); + return None; } - Ok(resp) => { - log::warn!("2PC status poll: coordinator returned {}", resp.status()); + }; + match execute_blocking_http(client, request, |resp| { + let success = resp.status().is_success(); + let body = resp.text().unwrap_or_default(); + Ok((success, body)) + }) { + Ok((true, body)) => Some(body.trim() == "commit"), + Ok((false, _)) => { + log::warn!("2PC status poll: coordinator returned non-success"); None } Err(e) => { @@ -1233,62 +1235,64 @@ impl InstanceCommon { } } - // Fire-and-forget: send COMMIT/ABORT to each participant. + // Send COMMIT/ABORT to each participant synchronously. // The coordinator log (written atomically with A's tx above) is the - // durability guarantee. If a send fails, recovery retransmits. - let replica_ctx = inst.replica_ctx().clone(); - handle.spawn(async move { - let client = replica_ctx.call_reducer_client.clone(); - let router = replica_ctx.call_reducer_router.clone(); - let auth_token = replica_ctx.call_reducer_auth_token.clone(); - - for (db_identity, prepare_id) in &prepared_participants { - let action = if committed { "commit" } else { "abort" }; - let base_url = match router.resolve_base_url(*db_identity).await { - Ok(url) => url, - Err(e) => { - log::error!("2PC {action}: failed to resolve base URL for {db_identity}: {e}"); - continue; - } - }; + // durability guarantee; if a send fails, recovery retransmits on restart. + let replica_ctx = inst.replica_ctx(); + let client = &replica_ctx.call_reducer_blocking_client; + let router = &replica_ctx.call_reducer_router; + let auth_token = &replica_ctx.call_reducer_auth_token; - let url = format!( - "{}/v1/database/{}/2pc/{}/{}", - base_url, - db_identity.to_hex(), - action, - prepare_id, - ); - let mut req = client.post(&url); - if let Some(ref token) = auth_token { - req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + for (db_identity, prepare_id) in &prepared_participants { + let action = if committed { "commit" } else { "abort" }; + let base_url = match router.resolve_base_url_blocking(*db_identity) { + Ok(url) => url, + Err(e) => { + log::error!("2PC {action}: failed to resolve base URL for {db_identity}: {e}"); + continue; } - - match req.send().await { - Ok(resp) if resp.status().is_success() => { - log::info!("2PC {action}: {prepare_id} on {db_identity}"); - // if committed { - // if let Err(e) = - // stdb.with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |del_tx| { - // Ok(del_tx.delete_st_2pc_coordinator_log(prepare_id)?) - // }) - // { - // log::warn!("delete_st_2pc_coordinator_log failed for {prepare_id}: {e}"); - // } - // } - } - Ok(resp) => { - log::error!( - "2PC {action}: failed for {prepare_id} on {db_identity}: status {}", - resp.status() - ); - } - Err(e) => { - log::error!("2PC {action}: transport error for {prepare_id} on {db_identity}: {e}"); + }; + let url = format!( + "{}/v1/database/{}/2pc/{}/{}", + base_url, + db_identity.to_hex(), + action, + prepare_id, + ); + let mut req = client.post(&url); + if let Some(token) = auth_token { + req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + } + let request = match req.build() { + Ok(r) => r, + Err(e) => { + log::error!("2PC {action}: failed to build request for {prepare_id} on {db_identity}: {e}"); + continue; + } + }; + match execute_blocking_http(client, request, |resp| Ok(resp.status())) { + Ok(status) if status.is_success() => { + log::info!("2PC {action}: {prepare_id} on {db_identity}"); + if committed { + if let Err(e) = replica_ctx + .subscriptions + .relational_db() + .with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |del_tx| { + Ok(del_tx.delete_st_2pc_coordinator_log(prepare_id)?) + }) + { + log::warn!("delete_st_2pc_coordinator_log failed for {prepare_id}: {e}"); + } } } + Ok(status) => { + log::error!("2PC {action}: failed for {prepare_id} on {db_identity}: status {status}"); + } + Err(e) => { + log::error!("2PC {action}: transport error for {prepare_id} on {db_identity}: {e}"); + } } - }); + } } if let Some(tx_id) = managed_global_tx_id { diff --git a/crates/core/src/replica_context.rs b/crates/core/src/replica_context.rs index fd342925b3a..122bde7fcb7 100644 --- a/crates/core/src/replica_context.rs +++ b/crates/core/src/replica_context.rs @@ -102,17 +102,63 @@ impl ReplicaContext { /// /// Used by the WASM executor thread and 2PC participant polling loop, which block their /// OS thread synchronously rather than yielding to tokio. + /// + /// `reqwest::blocking::Client::build()` internally creates and drops a mini tokio runtime, + /// which panics if called from inside an async context. We build it on a fresh OS thread + /// so it is safe to call from `async fn` at startup. pub fn new_call_reducer_blocking_client(config: &CallReducerOnDbConfig) -> reqwest::blocking::Client { - reqwest::blocking::Client::builder() - .tcp_keepalive(config.tcp_keepalive) - .pool_idle_timeout(config.pool_idle_timeout) - .pool_max_idle_per_host(config.pool_max_idle_per_host) - .timeout(config.request_timeout) - .http2_prior_knowledge() - .build() - .expect("failed to build call_reducer_on_db blocking HTTP client") + let tcp_keepalive = config.tcp_keepalive; + let pool_idle_timeout = config.pool_idle_timeout; + let pool_max_idle_per_host = config.pool_max_idle_per_host; + let timeout = config.request_timeout; + std::thread::scope(|s| { + s.spawn(move || { + reqwest::blocking::Client::builder() + .tcp_keepalive(tcp_keepalive) + .pool_idle_timeout(pool_idle_timeout) + .pool_max_idle_per_host(pool_max_idle_per_host) + .timeout(timeout) + .http2_prior_knowledge() + .build() + .expect("failed to build call_reducer_on_db blocking HTTP client") + }) + .join() + .expect("blocking client builder thread panicked") + }) } +} + +/// Execute a blocking reqwest request on a fresh OS thread, processing the response inside +/// that same thread. +/// +/// In debug builds, `reqwest 0.12` calls `wait::enter()` on every I/O operation +/// (`send`, `bytes`, `text`, …). That function creates and immediately drops a mini +/// tokio runtime as a nesting-check, which panics if the calling thread is already +/// inside a tokio `block_on` context (e.g. the `SingleCoreExecutor` WASM thread). +/// +/// By running both the send **and** all response reading inside a scoped OS thread that +/// has no tokio context, the assertion always passes. The closure `f` receives the +/// `Response` and must fully consume it (read body, extract headers, etc.) before +/// returning — do not let the `Response` escape the closure. +pub fn execute_blocking_http( + client: &reqwest::blocking::Client, + request: reqwest::blocking::Request, + f: F, +) -> reqwest::Result +where + F: FnOnce(reqwest::blocking::Response) -> reqwest::Result + Send, + T: Send, +{ + let client = client.clone(); + std::thread::scope(|s| { + s.spawn(move || client.execute(request).and_then(f)) + .join() + .expect("blocking HTTP thread panicked") + }) +} + +impl ReplicaContext { pub fn mint_global_tx_id(&self, start_ts: Timestamp) -> GlobalTxId { let nonce = self.tx_id_nonce.fetch_add(1, Ordering::Relaxed); GlobalTxId::new(start_ts, self.database.database_identity, nonce, 0) From 86e38d413b6a98a19e278e4ae1a07719f61ca830 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 14:32:10 +0530 Subject: [PATCH 16/33] update binding pointer --- crates/bindings-sys/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 56966cd84ab..630428ab653 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -1507,7 +1507,7 @@ pub fn call_reducer_on_db( ) -> Result<(u16, raw::BytesSource), (Errno, raw::BytesSource)> { let mut out = raw::BytesSource::INVALID; let status = unsafe { - raw::call_reducer_on_db( + raw::call_reducer_on_db_2pc( identity.as_ptr(), reducer_name.as_ptr(), reducer_name.len() as u32, From f46959414b9d0feb407c01bf5e6c6883c50be688 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 16:25:27 +0530 Subject: [PATCH 17/33] fix wound on blocking --- crates/core/src/host/instance_env.rs | 16 ++- crates/core/src/replica_context.rs | 54 ++++++- .../tests/smoketests/cross_db_chain.rs | 135 ++++++++++++++++++ 3 files changed, 197 insertions(+), 8 deletions(-) create mode 100644 crates/smoketests/tests/smoketests/cross_db_chain.rs diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 1265ac2548f..d65d87a7520 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -5,7 +5,7 @@ use crate::error::{DBError, DatastoreError, IndexError, NodesError}; use crate::host::global_tx::{GlobalTxRole, GlobalTxState}; use crate::host::module_host::{DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall}; use crate::host::wasm_common::TimingSpan; -use crate::replica_context::{execute_blocking_http, ReplicaContext}; +use crate::replica_context::{execute_blocking_http, execute_blocking_http_cancellable, HttpOutcome, ReplicaContext}; use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, ModuleSubscriptions}; use crate::subscription::module_subscription_manager::{from_tx_offset, TransactionOffset}; use crate::util::prometheus_handle::IntGaugeExt; @@ -1149,15 +1149,18 @@ impl InstanceEnv { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } - // Check for wound signal one more time before blocking. + // Check for wound signal one more time before blocking, then keep checking + // every 50 ms while the HTTP round-trip is in-flight. if self.replica_ctx.global_tx_manager.is_wounded(&tx_id) { return Err(self.wounded_tx_error(tx_id)); } let request = req.build().map_err(|e| NodesError::HttpError(e.to_string()))?; - let result = execute_blocking_http( + let manager = self.replica_ctx.global_tx_manager.clone(); + let outcome = execute_blocking_http_cancellable( &self.replica_ctx.call_reducer_blocking_client, request, + move || manager.is_wounded(&tx_id), |resp| { let status = resp.status().as_u16(); let prepare_id = resp @@ -1168,8 +1171,11 @@ impl InstanceEnv { let body = resp.bytes()?; Ok((status, body, prepare_id)) }, - ) - .map_err(|e| NodesError::HttpError(e.to_string())); + ); + let result = match outcome { + HttpOutcome::Done(r) => r.map_err(|e| NodesError::HttpError(e.to_string())), + HttpOutcome::Cancelled => return Err(self.wounded_tx_error(tx_id)), + }; WORKER_METRICS .cross_db_reducer_calls_total diff --git a/crates/core/src/replica_context.rs b/crates/core/src/replica_context.rs index 122bde7fcb7..6288fc92098 100644 --- a/crates/core/src/replica_context.rs +++ b/crates/core/src/replica_context.rs @@ -129,6 +129,54 @@ impl ReplicaContext { } +/// Outcome of [`execute_blocking_http_cancellable`]. +pub enum HttpOutcome { + Done(reqwest::Result), + Cancelled, +} + +/// Like [`execute_blocking_http`] but polls `should_cancel` every 50 ms while the HTTP +/// call is in-flight. If `should_cancel()` returns `true` the function returns +/// [`HttpOutcome::Cancelled`] immediately; the background HTTP thread is detached and +/// completes on its own (its result is silently discarded). +/// +/// All response reading must happen inside `f` — same rule as [`execute_blocking_http`]. +pub fn execute_blocking_http_cancellable( + client: &reqwest::blocking::Client, + request: reqwest::blocking::Request, + should_cancel: impl Fn() -> bool, + f: F, +) -> HttpOutcome +where + F: FnOnce(reqwest::blocking::Response) -> reqwest::Result + Send + 'static, + T: Send + 'static, +{ + use std::sync::mpsc; + let (tx, rx) = mpsc::channel::>(); + let client = client.clone(); + let handle = std::thread::spawn(move || { + let result = client.execute(request).and_then(f); + let _ = tx.send(result); + }); + let result = loop { + match rx.recv_timeout(std::time::Duration::from_millis(50)) { + Ok(result) => break Some(result), + Err(mpsc::RecvTimeoutError::Timeout) => { + if should_cancel() { + // Drop handle — thread is detached and its result discarded. + return HttpOutcome::Cancelled; + } + } + // Sender dropped without sending → thread panicked. + Err(mpsc::RecvTimeoutError::Disconnected) => break None, + } + }; + match result { + Some(r) => HttpOutcome::Done(r), + None => std::panic::resume_unwind(handle.join().unwrap_err()), + } +} + /// Execute a blocking reqwest request on a fresh OS thread, processing the response inside /// that same thread. /// @@ -147,14 +195,14 @@ pub fn execute_blocking_http( f: F, ) -> reqwest::Result where - F: FnOnce(reqwest::blocking::Response) -> reqwest::Result + Send, - T: Send, + F: FnOnce(reqwest::blocking::Response) -> reqwest::Result + Send + 'static, + T: Send + 'static, { let client = client.clone(); std::thread::scope(|s| { s.spawn(move || client.execute(request).and_then(f)) .join() - .expect("blocking HTTP thread panicked") + .unwrap_or_else(|e| std::panic::resume_unwind(e)) }) } diff --git a/crates/smoketests/tests/smoketests/cross_db_chain.rs b/crates/smoketests/tests/smoketests/cross_db_chain.rs new file mode 100644 index 00000000000..002ca4ff649 --- /dev/null +++ b/crates/smoketests/tests/smoketests/cross_db_chain.rs @@ -0,0 +1,135 @@ +use spacetimedb_smoketests::Smoketest; + +/// Module code shared by all three databases (A = initiator, B = relay, C = receiver). +/// +/// Tables: +/// - `PingLog(id auto_inc PK, message: String, priority: u32)` — records pings received. +/// +/// Reducers: +/// - `record_ping(payload)` — terminal: inserts payload into ping_log. +/// - `relay_ping(c_hex, payload)` — middle hop: forwards payload to C via `record_ping`, +/// then records a "relay:" entry locally so B's participation is verifiable. +/// - `chain_ping(b_hex, c_hex, message, priority)` — initiator: encodes a PingPayload and +/// calls `relay_ping` on B (which in turn calls `record_ping` on C), then records a +/// "chain:" entry locally. +/// +/// This exercises a two-hop cross-DB call chain (A → B → C). +const MODULE_CODE: &str = r#" +use spacetimedb::{log, ReducerContext, Table, Identity, SpacetimeType}; + +#[derive(SpacetimeType)] +pub struct PingPayload { + pub message: String, + pub priority: u32, +} + +#[spacetimedb::table(accessor = ping_log, public)] +pub struct PingLog { + #[primary_key] + #[auto_inc] + id: u64, + message: String, + priority: u32, +} + +/// Terminal hop: stores the payload in ping_log. +#[spacetimedb::reducer] +pub fn record_ping(ctx: &ReducerContext, payload: PingPayload) { + log::info!("record_ping: message={} priority={}", payload.message, payload.priority); + ctx.db.ping_log().insert(PingLog { id: 0, message: payload.message, priority: payload.priority }); +} + +/// Middle hop: forwards payload to C via `record_ping`, then records locally. +#[spacetimedb::reducer] +pub fn relay_ping(ctx: &ReducerContext, c_hex: String, payload: PingPayload) { + log::info!("relay_ping: forwarding to {c_hex}"); + let c = Identity::from_hex(&c_hex).expect("invalid C identity hex"); + let args = spacetimedb::spacetimedb_lib::bsatn::to_vec(&(PingPayload { message: payload.message.clone(), priority: payload.priority },)) + .expect("failed to encode args for record_ping"); + spacetimedb::remote_reducer::call_reducer_on_db(c, "record_ping", &args) + .unwrap_or_else(|e| panic!("relay_ping: call to C failed: {e}")); + ctx.db.ping_log().insert(PingLog { id: 0, message: format!("relay:{}", payload.message), priority: payload.priority }); +} + +/// Initiating hop: calls `relay_ping` on B (which calls `record_ping` on C), then records locally. +#[spacetimedb::reducer] +pub fn chain_ping(ctx: &ReducerContext, b_hex: String, c_hex: String, message: String, priority: u32) { + log::info!("chain_ping: starting A->B->C chain, message={message}"); + let b = Identity::from_hex(&b_hex).expect("invalid B identity hex"); + let payload = PingPayload { message: message.clone(), priority }; + let args = spacetimedb::spacetimedb_lib::bsatn::to_vec(&(c_hex, payload)) + .expect("failed to encode args for relay_ping"); + spacetimedb::remote_reducer::call_reducer_on_db(b, "relay_ping", &args) + .unwrap_or_else(|e| panic!("chain_ping: call to B failed: {e}")); + ctx.db.ping_log().insert(PingLog { id: 0, message: format!("chain:{message}"), priority }); +} +"#; + +fn query_ping_log(test: &Smoketest, db_identity: &str) -> String { + test.spacetime(&[ + "sql", + "--server", + &test.server_url, + db_identity, + "SELECT message, priority FROM ping_log ORDER BY id", + ]) + .unwrap_or_else(|e| panic!("sql query on {db_identity} failed: {e}")) +} + +/// Two-hop chain: A.chain_ping → B.relay_ping → C.record_ping. +/// +/// After the call: +/// - C's ping_log has the original message. +/// - B's ping_log has "relay:", confirming B was the relay. +/// - A's ping_log has "chain:", confirming A initiated the chain. +#[test] +fn test_cross_db_chain_call() { + let pid = std::process::id(); + let db_a_name = format!("chain-a-{pid}"); + let db_b_name = format!("chain-b-{pid}"); + let db_c_name = format!("chain-c-{pid}"); + + let mut test = Smoketest::builder().module_code(MODULE_CODE).autopublish(false).build(); + + // Publish C first (terminal), then B (relay), then A (initiator). + test.publish_module_named(&db_c_name, false) + .expect("failed to publish C"); + let db_c_identity = test.database_identity.clone().expect("C identity not set"); + + test.publish_module_named(&db_b_name, false) + .expect("failed to publish B"); + let db_b_identity = test.database_identity.clone().expect("B identity not set"); + + test.publish_module_named(&db_a_name, false) + .expect("failed to publish A"); + let db_a_identity = test.database_identity.clone().expect("A identity not set"); + + // Initiate the A → B → C chain. + test.call("chain_ping", &[&db_b_identity, &db_c_identity, "hello-chain", "7"]) + .expect("chain_ping call failed"); + + // C should have the original message. + let c_log = query_ping_log(&test, &db_c_identity); + assert!( + c_log.contains("hello-chain"), + "C ping_log should contain 'hello-chain' (original message), got:\n{c_log}" + ); + assert!( + c_log.contains('7'), + "C ping_log should contain priority 7, got:\n{c_log}" + ); + + // B should have "relay:hello-chain", confirming it was the relay hop. + let b_log = query_ping_log(&test, &db_b_identity); + assert!( + b_log.contains("relay:hello-chain"), + "B ping_log should contain 'relay:hello-chain', got:\n{b_log}" + ); + + // A should have "chain:hello-chain", confirming it initiated the chain. + let a_log = query_ping_log(&test, &db_a_identity); + assert!( + a_log.contains("chain:hello-chain"), + "A ping_log should contain 'chain:hello-chain', got:\n{a_log}" + ); +} From 78a1fd3e42cd771dd758b31688cd2f6b8f6ba3ad Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 16:29:51 +0530 Subject: [PATCH 18/33] reducer check timeout --- crates/core/src/replica_context.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/core/src/replica_context.rs b/crates/core/src/replica_context.rs index 6288fc92098..ab3d695efec 100644 --- a/crates/core/src/replica_context.rs +++ b/crates/core/src/replica_context.rs @@ -126,7 +126,6 @@ impl ReplicaContext { .expect("blocking client builder thread panicked") }) } - } /// Outcome of [`execute_blocking_http_cancellable`]. @@ -159,7 +158,7 @@ where let _ = tx.send(result); }); let result = loop { - match rx.recv_timeout(std::time::Duration::from_millis(50)) { + match rx.recv_timeout(std::time::Duration::from_millis(10)) { Ok(result) => break Some(result), Err(mpsc::RecvTimeoutError::Timeout) => { if should_cancel() { From 2766514f487f872ee4de3eb3b2e7cc1fa2c2ebb1 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 18:13:42 +0530 Subject: [PATCH 19/33] check for wounds for actively --- .../src/host/wasm_common/module_host_actor.rs | 57 ++++++++++++++----- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index c86db795ae2..057934c1bda 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -739,11 +739,18 @@ impl WasmModuleInstance { let return_value = event.reducer_return_value.clone(); let _ = prepared_tx.send((res, return_value)); - // Step 4: wait for coordinator's decision (B never aborts on its own). + // Step 4: wait for coordinator's decision, but abort early if the local + // transaction is wounded while this participant is prepared. let commit = !global_tx_id .map(|tx_id| replica_ctx.global_tx_manager.is_wounded(&tx_id)) .unwrap_or(false) - && Self::wait_for_2pc_decision(decision_rx, &prepare_id, coordinator_identity, &replica_ctx); + && Self::wait_for_2pc_decision( + decision_rx, + &prepare_id, + coordinator_identity, + global_tx_id, + &replica_ctx, + ); if commit { if let Some(tx_id) = global_tx_id { @@ -815,32 +822,54 @@ impl WasmModuleInstance { /// Wait for a 2PC COMMIT or ABORT decision for `prepare_id`. /// - /// First waits on `decision_rx` for up to 60 seconds. If no decision arrives, - /// switches to polling the coordinator's `GET /2pc/status/{prepare_id}` endpoint - /// every 5 seconds until a definitive answer is received. - /// - /// **B never aborts on its own** — ABORT is only returned when A explicitly says so. + /// First waits on `decision_rx` for up to 60 seconds, checking periodically for a + /// local wound signal. If no decision arrives, switches to polling the + /// coordinator's `GET /2pc/status/{prepare_id}` endpoint every 5 seconds until a + /// definitive answer is received or the local transaction is wounded. fn wait_for_2pc_decision( decision_rx: std::sync::mpsc::Receiver, prepare_id: &str, coordinator_identity: crate::identity::Identity, + global_tx_id: Option, replica_ctx: &std::sync::Arc, ) -> bool { - match decision_rx.recv_timeout(Duration::from_secs(60)) { - Ok(commit) => return commit, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { - log::warn!("2PC prepare_id={prepare_id}: no decision after 60s, polling coordinator"); + let decision_wait_deadline = std::time::Instant::now() + Duration::from_secs(20); + while std::time::Instant::now() < decision_wait_deadline { + if global_tx_id + .map(|tx_id| replica_ctx.global_tx_manager.is_wounded(&tx_id)) + .unwrap_or(false) + { + log::info!("2PC prepare_id={prepare_id}: local transaction wounded while waiting for decision"); + return false; } - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { - log::warn!("2PC prepare_id={prepare_id}: decision channel closed, polling coordinator"); + + let remaining = decision_wait_deadline.saturating_duration_since(std::time::Instant::now()); + let wait_slice = remaining.min(Duration::from_secs(1)); + match decision_rx.recv_timeout(wait_slice) { + Ok(commit) => return commit, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + log::warn!("2PC prepare_id={prepare_id}: decision channel closed, polling coordinator"); + break; + } } } + log::warn!("2PC prepare_id={prepare_id}: no decision after 60s, polling coordinator"); + let client = replica_ctx.call_reducer_blocking_client.clone(); let router = replica_ctx.call_reducer_router.clone(); let auth_token = replica_ctx.call_reducer_auth_token.clone(); let prepare_id_owned = prepare_id.to_owned(); loop { + if global_tx_id + .map(|tx_id| replica_ctx.global_tx_manager.is_wounded(&tx_id)) + .unwrap_or(false) + { + log::info!("2PC prepare_id={prepare_id}: local transaction wounded during status polling"); + return false; + } + let decision = Self::query_coordinator_status( &client, &router, @@ -850,7 +879,7 @@ impl WasmModuleInstance { ); match decision { Some(commit) => return commit, - None => std::thread::sleep(Duration::from_secs(5)), + None => std::thread::sleep(Duration::from_secs(1)), } } } From 5faf55a3f198e3e42953f5d98cd73f12aad510b8 Mon Sep 17 00:00:00 2001 From: Alessandro Asoni Date: Tue, 31 Mar 2026 14:51:23 +0200 Subject: [PATCH 20/33] Joshua: Log and return ok on missing request_id --- sdks/rust/src/db_connection.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sdks/rust/src/db_connection.rs b/sdks/rust/src/db_connection.rs index 4a3ee02a3e9..07ac4138c69 100644 --- a/sdks/rust/src/db_connection.rs +++ b/sdks/rust/src/db_connection.rs @@ -198,9 +198,13 @@ impl DbContextImpl { } => { let (reducer, callback) = { let mut inner = self.inner.lock().unwrap(); - inner.reducer_callbacks.pop_call_info(request_id).ok_or_else(|| { - InternalError::new(format!("Reducer result for unknown request_id {request_id}")) - })? + match inner.reducer_callbacks.pop_call_info(request_id) { + Some((reducer, callback)) => (reducer, callback), + None => { + log::error!("Reducer result for unknown request_id {request_id}"); + return Ok(()); + } + } }; let reducer_event = ReducerEvent { reducer, @@ -233,9 +237,13 @@ impl DbContextImpl { }; let (reducer, callback) = { let mut inner = self.inner.lock().unwrap(); - inner.reducer_callbacks.pop_call_info(request_id).ok_or_else(|| { - InternalError::new(format!("Reducer result for unknown request_id {request_id}")) - })? + match inner.reducer_callbacks.pop_call_info(request_id) { + Some((reducer, callback)) => (reducer, callback), + None => { + log::error!("Reducer result for unknown request_id {request_id}"); + return Ok(()); + } + } }; let reducer_event = ReducerEvent { From 5a836c2f3671267061b8bcb177538c3f9989edad Mon Sep 17 00:00:00 2001 From: Alessandro Asoni Date: Tue, 31 Mar 2026 15:18:05 +0200 Subject: [PATCH 21/33] Fix merge conflict errors --- tools/tpcc-runner/src/loader.rs | 518 -------------------------------- 1 file changed, 518 deletions(-) diff --git a/tools/tpcc-runner/src/loader.rs b/tools/tpcc-runner/src/loader.rs index 2cb7d58f664..559aa819d79 100644 --- a/tools/tpcc-runner/src/loader.rs +++ b/tools/tpcc-runner/src/loader.rs @@ -91,25 +91,6 @@ fn run_one_database(config: &LoadConfig, database_number: u32, topology: &Databa client.reset_tpcc().context("failed to reset tpcc data")?; } }); -<<<<<<< HEAD - if batch.len() >= batch_size { - client.queue_load_items(std::mem::take(&mut batch), pending, errors)?; - } - } - if !batch.is_empty() { - client.queue_load_items(batch, pending, errors)?; - } - Ok(()) -||||||| 2c04a393f - if batch.len() >= batch_size { - client.queue_load_items(std::mem::take(&mut batch), &pending, &errors)?; - } - } - if !batch.is_empty() { - client.queue_load_items(batch, &pending, &errors)?; - } - Ok(()) -======= let request = time!("build_load_request" { build_load_request(config, database_number, topology)? @@ -133,7 +114,6 @@ fn run_one_database(config: &LoadConfig, database_number: u32, topology: &Databa log::info!("tpcc load for database {database_identity} finished"); Ok(()) }) ->>>>>>> jdetter/tpcc } fn build_load_request( @@ -148,25 +128,6 @@ fn build_load_request( database_identities.push(topology.identity_for_database_number(database_number)?); } -<<<<<<< HEAD - while !warehouse_batch.is_empty() { - let split_at = warehouse_batch.len().min(batch_size); - let remainder = warehouse_batch.split_off(split_at); - let rows = std::mem::replace(&mut warehouse_batch, remainder); - client.queue_load_remote_warehouses(rows, pending, errors)?; - } - - Ok(()) -||||||| 2c04a393f - while !warehouse_batch.is_empty() { - let split_at = warehouse_batch.len().min(batch_size); - let remainder = warehouse_batch.split_off(split_at); - let rows = std::mem::replace(&mut warehouse_batch, remainder); - client.queue_load_remote_warehouses(rows, &pending, &errors)?; - } - - Ok(()) -======= Ok(TpccLoadConfigRequest { database_number, num_databases: config.num_databases, @@ -178,7 +139,6 @@ fn build_load_request( spacetimedb_uri: config.connection.uri.clone(), database_identities, }) ->>>>>>> jdetter/tpcc } fn wait_for_load_completion(client: &ModuleClient, database_identity: spacetimedb_sdk::Identity) -> Result<()> { @@ -187,305 +147,6 @@ fn wait_for_load_completion(client: &ModuleClient, database_identity: spacetimed loop { client.ensure_connected()?; -<<<<<<< HEAD - for d_id in 1..=DISTRICTS_PER_WAREHOUSE { - district_batch.push(District { - district_key: pack_district_key(w_id, d_id), - d_w_id: w_id, - d_id, - d_name: alpha_string(rng, 6, 10), - d_street_1: alpha_numeric_string(rng, 10, 20), - d_street_2: alpha_numeric_string(rng, 10, 20), - d_city: alpha_string(rng, 10, 20), - d_state: alpha_string(rng, 2, 2), - d_zip: zip_code(rng), - d_tax_bps: rng.random_range(0..=2_000), - d_ytd_cents: DISTRICT_YTD_CENTS, - d_next_o_id: CUSTOMERS_PER_DISTRICT + 1, - }); - } - } - - while !warehouse_batch.is_empty() { - let split_at = warehouse_batch.len().min(batch_size); - let remainder = warehouse_batch.split_off(split_at); - let rows = std::mem::replace(&mut warehouse_batch, remainder); - client.queue_load_warehouses(rows, pending, errors)?; - } - while !district_batch.is_empty() { - let split_at = district_batch.len().min(batch_size); - let remainder = district_batch.split_off(split_at); - let rows = std::mem::replace(&mut district_batch, remainder); - client.queue_load_districts(rows, pending, errors)?; - } - let _ = timestamp; - Ok(()) -} - -fn load_stock( - client: &ModuleClient, - database_number: u16, - warehouses_per_database: u16, - batch_size: usize, - rng: &mut StdRng, - pending: &Arc<(Mutex, Condvar)>, - errors: &Arc>>, -) -> Result<()> { - let mut batch = Vec::with_capacity(batch_size); - for w_id in warehouses_range(database_number, warehouses_per_database) { - for item_id in 1..=ITEMS { - batch.push(Stock { - stock_key: pack_stock_key(w_id, item_id), - s_w_id: w_id, - s_i_id: item_id, - s_quantity: rng.random_range(10..=100), - s_dist_01: alpha_string(rng, 24, 24), - s_dist_02: alpha_string(rng, 24, 24), - s_dist_03: alpha_string(rng, 24, 24), - s_dist_04: alpha_string(rng, 24, 24), - s_dist_05: alpha_string(rng, 24, 24), - s_dist_06: alpha_string(rng, 24, 24), - s_dist_07: alpha_string(rng, 24, 24), - s_dist_08: alpha_string(rng, 24, 24), - s_dist_09: alpha_string(rng, 24, 24), - s_dist_10: alpha_string(rng, 24, 24), - s_ytd: 0, - s_order_cnt: 0, - s_remote_cnt: 0, - s_data: maybe_with_original(rng, 26, 50), - }); - if batch.len() >= batch_size { - client.queue_load_stocks(std::mem::take(&mut batch), pending, errors)?; - } - } - } - if !batch.is_empty() { - client.queue_load_stocks(batch, pending, errors)?; - } - Ok(()) -} - -fn load_customers_history_orders( - client: &ModuleClient, - database_number: u16, - warehouses_per_database: u16, - batch_size: usize, - timestamp: Timestamp, - load_c_last: u32, - rng: &mut StdRng, - pending: &Arc<(Mutex, Condvar)>, - errors: &Arc>>, -) -> Result<()> { - let mut customer_batch = Vec::with_capacity(batch_size); - let mut history_batch = Vec::with_capacity(batch_size); - let mut order_batch = Vec::with_capacity(batch_size); - let mut new_order_batch = Vec::with_capacity(batch_size); - let mut order_line_batch = Vec::with_capacity(batch_size); - - for w_id in warehouses_range(database_number, warehouses_per_database) { - for d_id in 1..=DISTRICTS_PER_WAREHOUSE { - let mut permutation: Vec = (1..=CUSTOMERS_PER_DISTRICT).collect(); - permutation.shuffle(rng); - - for c_id in 1..=CUSTOMERS_PER_DISTRICT { - let credit = if rng.random_bool(0.10) { "BC" } else { "GC" }; - let last_name = if c_id <= 1_000 { - make_last_name(c_id - 1) - } else { - make_last_name(nurand(rng, 255, 0, 999, load_c_last)) - }; - customer_batch.push(Customer { - customer_key: pack_customer_key(w_id, d_id, c_id), - c_w_id: w_id, - c_d_id: d_id, - c_id, - c_first: alpha_string(rng, 8, 16), - c_middle: "OE".to_string(), - c_last: last_name, - c_street_1: alpha_numeric_string(rng, 10, 20), - c_street_2: alpha_numeric_string(rng, 10, 20), - c_city: alpha_string(rng, 10, 20), - c_state: alpha_string(rng, 2, 2), - c_zip: zip_code(rng), - c_phone: numeric_string(rng, 16, 16), - c_since: timestamp, - c_credit: credit.to_string(), - c_credit_lim_cents: CUSTOMER_CREDIT_LIMIT_CENTS, - c_discount_bps: rng.random_range(0..=5_000), - c_balance_cents: CUSTOMER_INITIAL_BALANCE_CENTS, - c_ytd_payment_cents: CUSTOMER_INITIAL_YTD_PAYMENT_CENTS, - c_payment_cnt: 1, - c_delivery_cnt: 0, - c_data: alpha_numeric_string(rng, 300, 500), - }); - history_batch.push(History { - history_id: 0, - h_c_id: c_id, - h_c_d_id: d_id, - h_c_w_id: w_id, - h_d_id: d_id, - h_w_id: w_id, - h_date: timestamp, - h_amount_cents: HISTORY_INITIAL_AMOUNT_CENTS, - h_data: alpha_numeric_string(rng, 12, 24), - }); - - if customer_batch.len() >= batch_size { - client.queue_load_customers(std::mem::take(&mut customer_batch), pending, errors)?; - } - if history_batch.len() >= batch_size { - client.queue_load_history(std::mem::take(&mut history_batch), pending, errors)?; - } -||||||| 2c04a393f - for d_id in 1..=DISTRICTS_PER_WAREHOUSE { - district_batch.push(District { - district_key: pack_district_key(w_id, d_id), - d_w_id: w_id, - d_id, - d_name: alpha_string(rng, 6, 10), - d_street_1: alpha_numeric_string(rng, 10, 20), - d_street_2: alpha_numeric_string(rng, 10, 20), - d_city: alpha_string(rng, 10, 20), - d_state: alpha_string(rng, 2, 2), - d_zip: zip_code(rng), - d_tax_bps: rng.random_range(0..=2_000), - d_ytd_cents: DISTRICT_YTD_CENTS, - d_next_o_id: CUSTOMERS_PER_DISTRICT + 1, - }); - } - } - - while !warehouse_batch.is_empty() { - let split_at = warehouse_batch.len().min(batch_size); - let remainder = warehouse_batch.split_off(split_at); - let rows = std::mem::replace(&mut warehouse_batch, remainder); - client.queue_load_warehouses(rows, &pending, &errors)?; - } - while !district_batch.is_empty() { - let split_at = district_batch.len().min(batch_size); - let remainder = district_batch.split_off(split_at); - let rows = std::mem::replace(&mut district_batch, remainder); - client.queue_load_districts(rows, &pending, &errors)?; - } - let _ = timestamp; - Ok(()) -} - -fn load_stock( - client: &ModuleClient, - database_number: u16, - warehouses_per_database: u16, - batch_size: usize, - rng: &mut StdRng, - pending: &Arc<(Mutex, Condvar)>, - errors: &Arc>>, -) -> Result<()> { - let mut batch = Vec::with_capacity(batch_size); - for w_id in warehouses_range(database_number, warehouses_per_database) { - for item_id in 1..=ITEMS { - batch.push(Stock { - stock_key: pack_stock_key(w_id, item_id), - s_w_id: w_id, - s_i_id: item_id, - s_quantity: rng.random_range(10..=100), - s_dist_01: alpha_string(rng, 24, 24), - s_dist_02: alpha_string(rng, 24, 24), - s_dist_03: alpha_string(rng, 24, 24), - s_dist_04: alpha_string(rng, 24, 24), - s_dist_05: alpha_string(rng, 24, 24), - s_dist_06: alpha_string(rng, 24, 24), - s_dist_07: alpha_string(rng, 24, 24), - s_dist_08: alpha_string(rng, 24, 24), - s_dist_09: alpha_string(rng, 24, 24), - s_dist_10: alpha_string(rng, 24, 24), - s_ytd: 0, - s_order_cnt: 0, - s_remote_cnt: 0, - s_data: maybe_with_original(rng, 26, 50), - }); - if batch.len() >= batch_size { - client.queue_load_stocks(std::mem::take(&mut batch), &pending, &errors)?; - } - } - } - if !batch.is_empty() { - client.queue_load_stocks(batch, &pending, &errors)?; - } - Ok(()) -} - -fn load_customers_history_orders( - client: &ModuleClient, - database_number: u16, - warehouses_per_database: u16, - batch_size: usize, - timestamp: Timestamp, - load_c_last: u32, - rng: &mut StdRng, - pending: &Arc<(Mutex, Condvar)>, - errors: &Arc>>, -) -> Result<()> { - let mut customer_batch = Vec::with_capacity(batch_size); - let mut history_batch = Vec::with_capacity(batch_size); - let mut order_batch = Vec::with_capacity(batch_size); - let mut new_order_batch = Vec::with_capacity(batch_size); - let mut order_line_batch = Vec::with_capacity(batch_size); - - for w_id in warehouses_range(database_number, warehouses_per_database) { - for d_id in 1..=DISTRICTS_PER_WAREHOUSE { - let mut permutation: Vec = (1..=CUSTOMERS_PER_DISTRICT).collect(); - permutation.shuffle(rng); - - for c_id in 1..=CUSTOMERS_PER_DISTRICT { - let credit = if rng.random_bool(0.10) { "BC" } else { "GC" }; - let last_name = if c_id <= 1_000 { - make_last_name(c_id - 1) - } else { - make_last_name(nurand(rng, 255, 0, 999, load_c_last)) - }; - customer_batch.push(Customer { - customer_key: pack_customer_key(w_id, d_id, c_id), - c_w_id: w_id, - c_d_id: d_id, - c_id, - c_first: alpha_string(rng, 8, 16), - c_middle: "OE".to_string(), - c_last: last_name, - c_street_1: alpha_numeric_string(rng, 10, 20), - c_street_2: alpha_numeric_string(rng, 10, 20), - c_city: alpha_string(rng, 10, 20), - c_state: alpha_string(rng, 2, 2), - c_zip: zip_code(rng), - c_phone: numeric_string(rng, 16, 16), - c_since: timestamp, - c_credit: credit.to_string(), - c_credit_lim_cents: CUSTOMER_CREDIT_LIMIT_CENTS, - c_discount_bps: rng.random_range(0..=5_000), - c_balance_cents: CUSTOMER_INITIAL_BALANCE_CENTS, - c_ytd_payment_cents: CUSTOMER_INITIAL_YTD_PAYMENT_CENTS, - c_payment_cnt: 1, - c_delivery_cnt: 0, - c_data: alpha_numeric_string(rng, 300, 500), - }); - history_batch.push(History { - history_id: 0, - h_c_id: c_id, - h_c_d_id: d_id, - h_c_w_id: w_id, - h_d_id: d_id, - h_w_id: w_id, - h_date: timestamp, - h_amount_cents: HISTORY_INITIAL_AMOUNT_CENTS, - h_data: alpha_numeric_string(rng, 12, 24), - }); - - if customer_batch.len() >= batch_size { - client.queue_load_customers(std::mem::take(&mut customer_batch), &pending, &errors)?; - } - if history_batch.len() >= batch_size { - client.queue_load_history(std::mem::take(&mut history_batch), &pending, &errors)?; - } -======= if let Some(state) = client.load_state() { let current_progress = ( state.status, @@ -511,116 +172,8 @@ fn load_customers_history_orders( state.next_order_id ); last_logged = Some(current_progress); ->>>>>>> jdetter/tpcc } -<<<<<<< HEAD - for o_id in 1..=CUSTOMERS_PER_DISTRICT { - let customer_id = permutation[(o_id - 1) as usize]; - let delivered = o_id < NEW_ORDER_START; - let order_line_count = rng.random_range(5..=15) as u8; - order_batch.push(OOrder { - order_key: pack_order_key(w_id, d_id, o_id), - o_w_id: w_id, - o_d_id: d_id, - o_id, - o_c_id: customer_id, - o_entry_d: timestamp, - o_carrier_id: if delivered { - Some(rng.random_range(1..=10)) - } else { - None - }, - o_ol_cnt: order_line_count, - o_all_local: true, - }); - if !delivered { - new_order_batch.push(NewOrder { - new_order_key: pack_order_key(w_id, d_id, o_id), - no_w_id: w_id, - no_d_id: d_id, - no_o_id: o_id, - }); - } - - for ol_number in 1..=order_line_count { - order_line_batch.push(OrderLine { - order_line_key: pack_order_line_key(w_id, d_id, o_id, ol_number), - ol_w_id: w_id, - ol_d_id: d_id, - ol_o_id: o_id, - ol_number, - ol_i_id: rng.random_range(1..=ITEMS), - ol_supply_w_id: w_id, - ol_delivery_d: if delivered { Some(timestamp) } else { None }, - ol_quantity: 5, - ol_amount_cents: if delivered { 0 } else { rng.random_range(1..=999_999) }, - ol_dist_info: alpha_string(rng, 24, 24), - }); - if order_line_batch.len() >= batch_size { - client.queue_load_order_lines(std::mem::take(&mut order_line_batch), pending, errors)?; - } - } - - if order_batch.len() >= batch_size { - client.queue_load_orders(std::mem::take(&mut order_batch), pending, errors)?; - } - if new_order_batch.len() >= batch_size { - client.queue_load_new_orders(std::mem::take(&mut new_order_batch), pending, errors)?; -||||||| 2c04a393f - for o_id in 1..=CUSTOMERS_PER_DISTRICT { - let customer_id = permutation[(o_id - 1) as usize]; - let delivered = o_id < NEW_ORDER_START; - let order_line_count = rng.random_range(5..=15) as u8; - order_batch.push(OOrder { - order_key: pack_order_key(w_id, d_id, o_id), - o_w_id: w_id, - o_d_id: d_id, - o_id, - o_c_id: customer_id, - o_entry_d: timestamp, - o_carrier_id: if delivered { - Some(rng.random_range(1..=10)) - } else { - None - }, - o_ol_cnt: order_line_count, - o_all_local: true, - }); - if !delivered { - new_order_batch.push(NewOrder { - new_order_key: pack_order_key(w_id, d_id, o_id), - no_w_id: w_id, - no_d_id: d_id, - no_o_id: o_id, - }); - } - - for ol_number in 1..=order_line_count { - order_line_batch.push(OrderLine { - order_line_key: pack_order_line_key(w_id, d_id, o_id, ol_number), - ol_w_id: w_id, - ol_d_id: d_id, - ol_o_id: o_id, - ol_number, - ol_i_id: rng.random_range(1..=ITEMS), - ol_supply_w_id: w_id, - ol_delivery_d: if delivered { Some(timestamp) } else { None }, - ol_quantity: 5, - ol_amount_cents: if delivered { 0 } else { rng.random_range(1..=999_999) }, - ol_dist_info: alpha_string(rng, 24, 24), - }); - if order_line_batch.len() >= batch_size { - client.queue_load_order_lines(std::mem::take(&mut order_line_batch), &pending, &errors)?; - } - } - - if order_batch.len() >= batch_size { - client.queue_load_orders(std::mem::take(&mut order_batch), &pending, &errors)?; - } - if new_order_batch.len() >= batch_size { - client.queue_load_new_orders(std::mem::take(&mut new_order_batch), &pending, &errors)?; -======= match state.status { TpccLoadStatus::Complete => return Ok(()), TpccLoadStatus::Failed => { @@ -631,82 +184,11 @@ fn load_customers_history_orders( .last_error .unwrap_or_else(|| "load failed without an error message".to_string()) ) ->>>>>>> jdetter/tpcc } TpccLoadStatus::Idle | TpccLoadStatus::Running => {} } } -<<<<<<< HEAD - if !customer_batch.is_empty() { - client.queue_load_customers(customer_batch, pending, errors)?; - } - if !history_batch.is_empty() { - client.queue_load_history(history_batch, pending, errors)?; - } - if !order_batch.is_empty() { - client.queue_load_orders(order_batch, pending, errors)?; - } - if !new_order_batch.is_empty() { - client.queue_load_new_orders(new_order_batch, pending, errors)?; - } - if !order_line_batch.is_empty() { - client.queue_load_order_lines(order_line_batch, pending, errors)?; - } - - Ok(()) -} - -fn wait_for_pending(pending: &Arc<(Mutex, Condvar)>) { - let (lock, cvar) = pending.as_ref(); - let mut guard = lock.lock().unwrap(); - while *guard > 0 { - guard = cvar.wait(guard).unwrap(); - } -} - -fn take_first_error(errors: &Arc>>) -> Result<()> { - let mut guard = errors.lock().unwrap(); - if let Some(err) = guard.take() { - Err(err) - } else { - Ok(()) -||||||| 2c04a393f - if !customer_batch.is_empty() { - client.queue_load_customers(customer_batch, &pending, &errors)?; - } - if !history_batch.is_empty() { - client.queue_load_history(history_batch, &pending, &errors)?; - } - if !order_batch.is_empty() { - client.queue_load_orders(order_batch, &pending, &errors)?; - } - if !new_order_batch.is_empty() { - client.queue_load_new_orders(new_order_batch, &pending, &errors)?; - } - if !order_line_batch.is_empty() { - client.queue_load_order_lines(order_line_batch, &pending, &errors)?; - } - - Ok(()) -} - -fn wait_for_pending(pending: &Arc<(Mutex, Condvar)>) { - let (lock, cvar) = pending.as_ref(); - let mut guard = lock.lock().unwrap(); - while *guard > 0 { - guard = cvar.wait(guard).unwrap(); - } -} - -fn take_first_error(errors: &Arc>>) -> Result<()> { - let mut guard = errors.lock().unwrap(); - if let Some(err) = guard.take() { - Err(err) - } else { - Ok(()) -======= thread::sleep(Duration::from_millis(250)); ->>>>>>> jdetter/tpcc } } From 56d69ffb4f0a57d6d0d40cdfcb4cdf85d7405f85 Mon Sep 17 00:00:00 2001 From: Alessandro Asoni Date: Tue, 31 Mar 2026 15:18:47 +0200 Subject: [PATCH 22/33] Add client retries where reducers error --- tools/tpcc-runner/src/driver.rs | 379 ++++++++++++++++++++------------ 1 file changed, 237 insertions(+), 142 deletions(-) diff --git a/tools/tpcc-runner/src/driver.rs b/tools/tpcc-runner/src/driver.rs index 13703f29846..cb937f9d8b3 100644 --- a/tools/tpcc-runner/src/driver.rs +++ b/tools/tpcc-runner/src/driver.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, bail, Context, Result}; use rand::{rngs::StdRng, Rng, SeedableRng}; use std::collections::BTreeMap; use std::fs; +use std::future::Future; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; @@ -23,6 +24,9 @@ use crate::summary::{ use crate::topology::DatabaseTopology; use crate::tpcc::*; +const REDUCER_CALL_MAX_ATTEMPTS: u32 = 3; +const REDUCER_RETRY_DELAY_MS: u64 = 250; + struct TerminalRuntime { config: DriverConfig, client: Arc, @@ -48,6 +52,16 @@ struct TransactionContext<'a> { request_ids: &'a AtomicU64, } +enum ReducerCallOutcome { + Succeeded(T), + Failed(anyhow::Error), +} + +enum NewOrderExecution { + Committed, + RolledBack(String), +} + pub async fn run(config: DriverConfig) -> Result<()> { let (config, schedule) = resolve_driver_setup(config).await?; let run_id = schedule.run_id.clone(); @@ -279,76 +293,40 @@ async fn execute_transaction( started_ms: u64, ) -> Result { match kind { - TransactionKind::NewOrder => { - execute_new_order( - context.client, - context.config.warehouse_count, - context.assignment, - context.constants, - rng, - started_ms, - ) - .await - } - TransactionKind::Payment => { - execute_payment( - context.client, - context.config.warehouse_count, - context.assignment, - context.constants, - rng, - started_ms, - ) - .await - } - TransactionKind::OrderStatus => { - execute_order_status(context.client, context.assignment, context.constants, rng, started_ms).await - } - TransactionKind::Delivery => { - execute_delivery( - context.client, - context.run_id, - context.driver_id, - context.assignment, - context.request_ids, - rng, - started_ms, - ) - .await - } - TransactionKind::StockLevel => execute_stock_level(context.client, context.assignment, rng, started_ms).await, + TransactionKind::NewOrder => execute_new_order(context, rng, started_ms).await, + TransactionKind::Payment => execute_payment(context, rng, started_ms).await, + TransactionKind::OrderStatus => execute_order_status(context, rng, started_ms).await, + TransactionKind::Delivery => execute_delivery(context, rng, started_ms).await, + TransactionKind::StockLevel => execute_stock_level(context, rng, started_ms).await, } } async fn execute_new_order( - client: &ModuleClient, - warehouse_count: u32, - assignment: &TerminalAssignment, - constants: &RunConstants, + context: &TransactionContext<'_>, rng: &mut StdRng, started_ms: u64, ) -> Result { - let customer_id = customer_id(rng, constants); + let customer_id = customer_id(rng, context.constants); let line_count = rng.random_range(5..=15); let invalid_line = rng.random_bool(0.01); let mut order_lines = Vec::with_capacity(line_count); let mut remote_order_line_count = 0u32; for idx in 0..line_count { - let remote = warehouse_count > 1 && rng.random_bool(0.01); + let remote = context.config.warehouse_count > 1 && rng.random_bool(0.01); let supply_w_id = if remote { remote_order_line_count += 1; - let mut remote = assignment.warehouse_id; - while remote == assignment.warehouse_id { - remote = rng.random_range(1..=warehouse_count); + let mut remote = context.assignment.warehouse_id; + while remote == context.assignment.warehouse_id { + remote = rng.random_range(1..=context.config.warehouse_count); } remote } else { - assignment.warehouse_id + context.assignment.warehouse_id }; let item_id = if invalid_line && idx + 1 == line_count { ITEMS + 1 } else { - item_id(rng, constants) + item_id(rng, context.constants) }; order_lines.push(NewOrderLineInput { item_id, @@ -357,19 +335,31 @@ async fn execute_new_order( }); } - let result = client - .new_order_async( - assignment.warehouse_id, - assignment.district_id, - customer_id, - order_lines, - ) - .await?; + let result = retry_reducer_call(context, "new_order", || { + let order_lines = order_lines.clone(); + async move { + match context + .client + .new_order_async( + context.assignment.warehouse_id, + context.assignment.district_id, + customer_id, + order_lines, + ) + .await? + { + Ok(_) => Ok(NewOrderExecution::Committed), + Err(message) if invalid_line => Ok(NewOrderExecution::RolledBack(message)), + Err(message) => Err(anyhow!("new_order failed: {}", message)), + } + } + }) + .await; let finished_ms = crate::summary::now_millis(); match result { - Ok(_) => Ok(TransactionRecord { + ReducerCallOutcome::Succeeded(NewOrderExecution::Committed) => Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: assignment.terminal_id, + terminal_id: context.assignment.terminal_id, kind: TransactionKind::NewOrder, success: true, latency_ms: finished_ms.saturating_sub(started_ms), @@ -380,9 +370,9 @@ async fn execute_new_order( remote_order_line_count, detail: None, }), - Err(message) if invalid_line => Ok(TransactionRecord { + ReducerCallOutcome::Succeeded(NewOrderExecution::RolledBack(message)) => Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: assignment.terminal_id, + terminal_id: context.assignment.terminal_id, kind: TransactionKind::NewOrder, success: false, latency_ms: finished_ms.saturating_sub(started_ms), @@ -393,177 +383,282 @@ async fn execute_new_order( remote_order_line_count, detail: Some(message), }), - Err(message) => bail!( - "unexpected new_order failure for terminal {}: {}", - assignment.terminal_id, - message - ), + ReducerCallOutcome::Failed(err) => Ok(TransactionRecord { + timestamp_ms: finished_ms, + terminal_id: context.assignment.terminal_id, + kind: TransactionKind::NewOrder, + success: false, + latency_ms: finished_ms.saturating_sub(started_ms), + rollback: false, + remote: false, + by_last_name: false, + order_line_count: line_count as u32, + remote_order_line_count, + detail: Some(format!("{err:#}")), + }), } } async fn execute_payment( - client: &ModuleClient, - warehouse_count: u32, - assignment: &TerminalAssignment, - constants: &RunConstants, + context: &TransactionContext<'_>, rng: &mut StdRng, started_ms: u64, ) -> Result { - let remote = warehouse_count > 1 && rng.random_bool(0.15); + let remote = context.config.warehouse_count > 1 && rng.random_bool(0.15); let c_w_id = if remote { - let mut other = assignment.warehouse_id; - while other == assignment.warehouse_id { - other = rng.random_range(1..=warehouse_count); + let mut other = context.assignment.warehouse_id; + while other == context.assignment.warehouse_id { + other = rng.random_range(1..=context.config.warehouse_count); } other } else { - assignment.warehouse_id + context.assignment.warehouse_id }; let c_d_id = if remote { rng.random_range(1..=DISTRICTS_PER_WAREHOUSE) } else { - assignment.district_id + context.assignment.district_id }; let by_last_name = rng.random_bool(0.60); let selector = if by_last_name { - CustomerSelector::ByLastName(customer_last_name(rng, constants)) + CustomerSelector::ByLastName(customer_last_name(rng, context.constants)) } else { - CustomerSelector::ById(customer_id(rng, constants)) + CustomerSelector::ById(customer_id(rng, context.constants)) }; let amount_cents = rng.random_range(100..=500_000); - let finished = expect_ok( - "payment", - client - .payment_async( - assignment.warehouse_id, - assignment.district_id, - c_w_id, - c_d_id, - selector, - amount_cents, - ) - .await, - )?; - let _ = finished; + let result = retry_reducer_call(context, "payment", || { + let selector = selector.clone(); + async move { + let _ = expect_ok( + "payment", + context + .client + .payment_async( + context.assignment.warehouse_id, + context.assignment.district_id, + c_w_id, + c_d_id, + selector, + amount_cents, + ) + .await, + )?; + Ok(()) + } + }) + .await; let finished_ms = crate::summary::now_millis(); + let (success, detail) = match result { + ReducerCallOutcome::Succeeded(()) => (true, None), + ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), + }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: assignment.terminal_id, + terminal_id: context.assignment.terminal_id, kind: TransactionKind::Payment, - success: true, + success, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote, by_last_name, order_line_count: 0, remote_order_line_count: 0, - detail: None, + detail, }) } async fn execute_order_status( - client: &ModuleClient, - assignment: &TerminalAssignment, - constants: &RunConstants, + context: &TransactionContext<'_>, rng: &mut StdRng, started_ms: u64, ) -> Result { let by_last_name = rng.random_bool(0.60); let selector = if by_last_name { - CustomerSelector::ByLastName(customer_last_name(rng, constants)) + CustomerSelector::ByLastName(customer_last_name(rng, context.constants)) } else { - CustomerSelector::ById(customer_id(rng, constants)) + CustomerSelector::ById(customer_id(rng, context.constants)) }; - let _ = expect_ok( - "order_status", - client - .order_status_async(assignment.warehouse_id, assignment.district_id, selector) - .await, - )?; + let result = retry_reducer_call(context, "order_status", || { + let selector = selector.clone(); + async move { + let _ = expect_ok( + "order_status", + context + .client + .order_status_async( + context.assignment.warehouse_id, + context.assignment.district_id, + selector, + ) + .await, + )?; + Ok(()) + } + }) + .await; let finished_ms = crate::summary::now_millis(); + let (success, detail) = match result { + ReducerCallOutcome::Succeeded(()) => (true, None), + ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), + }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: assignment.terminal_id, + terminal_id: context.assignment.terminal_id, kind: TransactionKind::OrderStatus, - success: true, + success, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote: false, by_last_name, order_line_count: 0, remote_order_line_count: 0, - detail: None, + detail, }) } async fn execute_delivery( - client: &ModuleClient, - run_id: &str, - driver_id: &str, - assignment: &TerminalAssignment, - request_ids: &AtomicU64, + context: &TransactionContext<'_>, rng: &mut StdRng, started_ms: u64, ) -> Result { - let request_id = request_ids.fetch_add(1, Ordering::Relaxed); - let _ = expect_ok( - "queue_delivery", - client - .queue_delivery_async( - run_id.to_string(), - driver_id.to_string(), - assignment.terminal_id, - request_id, - assignment.warehouse_id, - rng.random_range(1..=10), - ) - .await, - )?; + let request_id = context.request_ids.fetch_add(1, Ordering::Relaxed); + let carrier_id = rng.random_range(1..=10); + let result = retry_reducer_call(context, "queue_delivery", || async move { + let _ = expect_ok( + "queue_delivery", + context + .client + .queue_delivery_async( + context.run_id.to_string(), + context.driver_id.to_string(), + context.assignment.terminal_id, + request_id, + context.assignment.warehouse_id, + carrier_id, + ) + .await, + )?; + Ok(()) + }) + .await; let finished_ms = crate::summary::now_millis(); + let (success, detail) = match result { + ReducerCallOutcome::Succeeded(()) => (true, None), + ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), + }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: assignment.terminal_id, + terminal_id: context.assignment.terminal_id, kind: TransactionKind::Delivery, - success: true, + success, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote: false, by_last_name: false, order_line_count: 0, remote_order_line_count: 0, - detail: None, + detail, }) } async fn execute_stock_level( - client: &ModuleClient, - assignment: &TerminalAssignment, + context: &TransactionContext<'_>, rng: &mut StdRng, started_ms: u64, ) -> Result { let threshold = rng.random_range(10..=20); - let _ = expect_ok( - "stock_level", - client - .stock_level_async(assignment.warehouse_id, assignment.district_id, threshold) - .await, - )?; + let result = retry_reducer_call(context, "stock_level", || async move { + let _ = expect_ok( + "stock_level", + context + .client + .stock_level_async( + context.assignment.warehouse_id, + context.assignment.district_id, + threshold, + ) + .await, + )?; + Ok(()) + }) + .await; let finished_ms = crate::summary::now_millis(); + let (success, detail) = match result { + ReducerCallOutcome::Succeeded(()) => (true, None), + ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), + }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: assignment.terminal_id, + terminal_id: context.assignment.terminal_id, kind: TransactionKind::StockLevel, - success: true, + success, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote: false, by_last_name: false, order_line_count: 0, remote_order_line_count: 0, - detail: None, + detail, }) } +async fn retry_reducer_call( + context: &TransactionContext<'_>, + reducer_name: &'static str, + mut call: F, +) -> ReducerCallOutcome +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut last_error = None; + + for attempt in 1..=REDUCER_CALL_MAX_ATTEMPTS { + match call().await { + Ok(value) => { + if attempt > 1 { + log::info!( + "driver {} terminal {} reducer {} succeeded on attempt {}/{}", + context.driver_id, + context.assignment.terminal_id, + reducer_name, + attempt, + REDUCER_CALL_MAX_ATTEMPTS + ); + } + return ReducerCallOutcome::Succeeded(value); + } + Err(err) => { + if attempt == REDUCER_CALL_MAX_ATTEMPTS { + log::error!( + "driver {} terminal {} reducer {} failed after {} attempt(s): {err:#}", + context.driver_id, + context.assignment.terminal_id, + reducer_name, + REDUCER_CALL_MAX_ATTEMPTS + ); + last_error = Some(err); + break; + } + + log::warn!( + "driver {} terminal {} reducer {} failed on attempt {}/{}: {err:#}; retrying in {}ms", + context.driver_id, + context.assignment.terminal_id, + reducer_name, + attempt, + REDUCER_CALL_MAX_ATTEMPTS, + REDUCER_RETRY_DELAY_MS + ); + tokio::time::sleep(Duration::from_millis(REDUCER_RETRY_DELAY_MS)).await; + last_error = Some(err); + } + } + } + + ReducerCallOutcome::Failed(last_error.unwrap_or_else(|| anyhow!("{} failed without an error", reducer_name))) +} + async fn resolve_driver_setup(config: DriverConfig) -> Result<(DriverConfig, RunSchedule)> { if let Some(coordinator_url) = &config.coordinator_url { const REGISTER_ATTEMPTS: u32 = 5; From d19152e3a47facaa364592a380c8cce7b76dad21 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 19:50:25 +0530 Subject: [PATCH 23/33] remove commit-ack --- crates/core/src/db/relational_db.rs | 1 - crates/core/src/host/instance_env.rs | 14 ++-- crates/core/src/host/module_host.rs | 13 ++-- .../src/host/wasm_common/module_host_actor.rs | 72 +++++++++---------- 4 files changed, 47 insertions(+), 53 deletions(-) diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 79c0d9593de..12837a34ad5 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -62,7 +62,6 @@ use std::num::NonZeroUsize; use std::ops::RangeBounds; use std::sync::Arc; use tokio::sync::watch; -use tokio::task::spawn_blocking; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; pub use super::snapshot::SnapshotWorker; diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index d65d87a7520..7be2ee47ad0 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1071,15 +1071,11 @@ impl InstanceEnv { req = req.header(TX_ID_HEADER, tx_id.to_string()); } let request = req.build().map_err(|e| NodesError::HttpError(e.to_string()))?; - let result = execute_blocking_http( - &self.replica_ctx.call_reducer_blocking_client, - request, - |resp| { - let status = resp.status().as_u16(); - let body = resp.bytes()?; - Ok((status, body)) - }, - ) + let result = execute_blocking_http(&self.replica_ctx.call_reducer_blocking_client, request, |resp| { + let status = resp.status().as_u16(); + let body = resp.bytes()?; + Ok((status, body)) + }) .map_err(|e| NodesError::HttpError(e.to_string())); WORKER_METRICS diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 3819fa09022..4424899f3ec 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2226,12 +2226,13 @@ impl ModuleHost { /// Delete a coordinator log entry for `prepare_id`. /// Called when B has confirmed it committed, so A can stop retransmitting. - pub fn ack_2pc_coordinator_commit(&self, prepare_id: &str) -> Result<(), anyhow::Error> { - let db = self.relational_db().clone(); - db.with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |tx| { - tx.delete_st_2pc_coordinator_log(prepare_id) - .map_err(anyhow::Error::from) - }) + pub fn ack_2pc_coordinator_commit(&self, _prepare_id: &str) -> Result<(), anyhow::Error> { + let _db = self.relational_db().clone(); + // db.with_auto_commit::<_, _, anyhow::Error>(Workload::Internal, |tx| { + // tx.delete_st_2pc_coordinator_log(prepare_id) + // .map_err(anyhow::Error::from) + // }) + Ok(()) } /// Check whether `prepare_id` is present in the coordinator log of this database. diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 057934c1bda..26d524f0f21 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -405,42 +405,49 @@ impl WasmModuleHostActor { /// /// Called AFTER B's commit is durable. Fire-and-forget: failure is tolerated because /// `recover_2pc_coordinator` on A will retransmit COMMIT on restart. -fn send_ack_commit_to_coordinator( +fn _send_ack_commit_to_coordinator( client: &reqwest::blocking::Client, router: &std::sync::Arc, auth_token: &Option, coordinator_identity: crate::identity::Identity, prepare_id: &str, ) { - let base_url = match router.resolve_base_url_blocking(coordinator_identity) { - Ok(url) => url, - Err(e) => { - log::warn!("2PC ack-commit: cannot resolve coordinator URL: {e}"); - return; - } - }; - let url = format!( - "{}/v1/database/{}/2pc/ack-commit/{}", - base_url, - coordinator_identity.to_hex(), - prepare_id, - ); - let mut req = client.post(&url); - if let Some(token) = auth_token { - req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); - } - let Ok(request) = req.build() else { return }; - match execute_blocking_http(client, request, |resp| Ok(resp.status())) { - Ok(status) if status.is_success() => { - log::info!("2PC ack-commit: notified coordinator for {prepare_id}"); - } - Ok(status) => { - log::warn!("2PC ack-commit: coordinator returned {status} for {prepare_id}"); + let client = client.clone(); + let router = router.clone(); + let auth_token = auth_token.clone(); + let prepare_id = prepare_id.to_owned(); + + std::thread::spawn(move || { + let base_url = match router.resolve_base_url_blocking(coordinator_identity) { + Ok(url) => url, + Err(e) => { + log::warn!("2PC ack-commit: cannot resolve coordinator URL: {e}"); + return; + } + }; + let url = format!( + "{}/v1/database/{}/2pc/ack-commit/{}", + base_url, + coordinator_identity.to_hex(), + prepare_id, + ); + let mut req = client.post(&url); + if let Some(token) = &auth_token { + req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } - Err(e) => { - log::warn!("2PC ack-commit: transport error for {prepare_id}: {e}"); + let Ok(request) = req.build() else { return }; + match execute_blocking_http(&client, request, |resp| Ok(resp.status())) { + Ok(status) if status.is_success() => { + log::info!("2PC ack-commit: notified coordinator for {prepare_id}"); + } + Ok(status) => { + log::warn!("2PC ack-commit: coordinator returned {status} for {prepare_id}"); + } + Err(e) => { + log::warn!("2PC ack-commit: transport error for {prepare_id}: {e}"); + } } - } + }); } fn wounded_status(replica_ctx: &ReplicaContext, tx_id: spacetimedb_lib::GlobalTxId) -> EventStatus { @@ -778,15 +785,6 @@ impl WasmModuleInstance { }); } - // Notify coordinator that B has committed so it can delete its coordinator log entry. - // Best-effort: if this fails, coordinator's recover_2pc_coordinator will retry on restart. - send_ack_commit_to_coordinator( - &replica_ctx.call_reducer_blocking_client, - &replica_ctx.call_reducer_router, - &replica_ctx.call_reducer_auth_token, - coordinator_identity, - &prepare_id, - ); if let Some(tx_id) = global_tx_id { replica_ctx .global_tx_manager From 555a86243e5a95ae7b9055b8282574d5650a3aa2 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 31 Mar 2026 22:03:04 +0530 Subject: [PATCH 24/33] test script --- Cargo.lock | 7 ++ Cargo.toml | 1 + modules/chain-call-repro/Cargo.toml | 11 ++ modules/chain-call-repro/run.sh | 158 ++++++++++++++++++++++++++++ modules/chain-call-repro/src/lib.rs | 118 +++++++++++++++++++++ 5 files changed, 295 insertions(+) create mode 100644 modules/chain-call-repro/Cargo.toml create mode 100755 modules/chain-call-repro/run.sh create mode 100644 modules/chain-call-repro/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 4a1c238f49d..5debdc2594b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -842,6 +842,13 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chain-call-repro" +version = "0.0.0" +dependencies = [ + "spacetimedb 2.1.0", +] + [[package]] name = "check-license-symlinks" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 87a2c459245..7d82ec900e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/chain-call-repro", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "templates/keynote-2/spacetimedb-rust-client", diff --git a/modules/chain-call-repro/Cargo.toml b/modules/chain-call-repro/Cargo.toml new file mode 100644 index 00000000000..37e966784f4 --- /dev/null +++ b/modules/chain-call-repro/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "chain-call-repro" +version = "0.0.0" +edition.workspace = true + +[lib] +crate-type = ["cdylib"] +bench = false + +[dependencies] +spacetimedb = { path = "../../crates/bindings", features = ["unstable"] } diff --git a/modules/chain-call-repro/run.sh b/modules/chain-call-repro/run.sh new file mode 100755 index 00000000000..454341436a2 --- /dev/null +++ b/modules/chain-call-repro/run.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +SERVER="${SPACETIME_SERVER:-local}" +A_CLIENTS="${A_CLIENTS:-4}" +B_CLIENTS="${B_CLIENTS:-4}" +ITERATIONS="${ITERATIONS:-25}" +BURN_ITERS="${BURN_ITERS:-0}" +RUN_ID="$(date +%Y%m%d%H%M%S)-$$" +DB_A="independent-repro-a-${RUN_ID}" +DB_B="independent-repro-b-${RUN_ID}" +DB_C="independent-repro-c-${RUN_ID}" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +publish_db() { + local db_name="$1" + local output + local identity + + output="$(cd "$SCRIPT_DIR" && spacetime publish --server "$SERVER" --clear-database -y "$db_name")" + printf '%s\n' "$output" >&2 + + identity="$( + printf '%s\n' "$output" \ + | grep -Eo 'identity: [0-9a-fA-F]+' \ + | sed 's/^identity: //' \ + | tail -n1 + )" + if [[ -z "$identity" ]]; then + echo "failed to parse identity from publish output for $db_name" >&2 + return 1 + fi + + printf '%s\n' "$identity" +} + +run_a_client() { + local client_id="$1" + local failures=0 + local seq + + for ((seq = 1; seq <= ITERATIONS; seq++)); do + if ! ( + cd "$SCRIPT_DIR" && + spacetime call --server "$SERVER" -- "$DB_A_ID" call_b_from_a \ + "$DB_B_ID" \ + "a-client-${client_id}" \ + "$seq" \ + "a-msg-client-${client_id}-seq-${seq}" \ + "$BURN_ITERS" + ) >"$TMP_DIR/a-client-${client_id}-seq-${seq}.log" 2>&1; then + failures=$((failures + 1)) + fi + done + + printf '%s\n' "$failures" >"$TMP_DIR/a-client-${client_id}.failures" +} + +run_b_client() { + local client_id="$1" + local failures=0 + local seq + + for ((seq = 1; seq <= ITERATIONS; seq++)); do + if ! ( + cd "$SCRIPT_DIR" && + spacetime call --server "$SERVER" -- "$DB_B_ID" call_c_from_b \ + "$DB_C_ID" \ + "b-client-${client_id}" \ + "$seq" \ + "b-msg-client-${client_id}-seq-${seq}" \ + "$BURN_ITERS" + ) >"$TMP_DIR/b-client-${client_id}-seq-${seq}.log" 2>&1; then + failures=$((failures + 1)) + fi + done + + printf '%s\n' "$failures" >"$TMP_DIR/b-client-${client_id}.failures" +} + +echo "Publishing independent-call repro module to A, B, and C on server '$SERVER'..." +DB_C_ID="$(publish_db "$DB_C")" +DB_B_ID="$(publish_db "$DB_B")" +DB_A_ID="$(publish_db "$DB_A")" + +echo "A identity: $DB_A_ID" +echo "B identity: $DB_B_ID" +echo "C identity: $DB_C_ID" +echo "Starting $A_CLIENTS A-clients and $B_CLIENTS B-clients with $ITERATIONS calls each..." + +for ((client_id = 1; client_id <= A_CLIENTS; client_id++)); do + run_a_client "$client_id" & +done +for ((client_id = 1; client_id <= B_CLIENTS; client_id++)); do + run_b_client "$client_id" & +done +wait + +A_FAILURES=0 +for ((client_id = 1; client_id <= A_CLIENTS; client_id++)); do + client_failures="$(cat "$TMP_DIR/a-client-${client_id}.failures")" + A_FAILURES=$((A_FAILURES + client_failures)) +done + +B_FAILURES=0 +for ((client_id = 1; client_id <= B_CLIENTS; client_id++)); do + client_failures="$(cat "$TMP_DIR/b-client-${client_id}.failures")" + B_FAILURES=$((B_FAILURES + client_failures)) +done + +A_SUCCESSES=$((A_CLIENTS * ITERATIONS - A_FAILURES)) +B_SUCCESSES=$((B_CLIENTS * ITERATIONS - B_FAILURES)) +TOTAL_FAILURES=$((A_FAILURES + B_FAILURES)) + +echo "Successful A->B calls: $A_SUCCESSES" +echo "Failed A->B calls: $A_FAILURES" +echo "Successful B->C calls: $B_SUCCESSES" +echo "Failed B->C calls: $B_FAILURES" + +if [[ "$A_SUCCESSES" -gt 0 ]]; then + (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_A_ID" assert_kind_count sent_to_b "$A_SUCCESSES") + (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_B_ID" assert_kind_count recv_from_a "$A_SUCCESSES") +fi + +if [[ "$B_SUCCESSES" -gt 0 ]]; then + (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_B_ID" assert_kind_count sent_to_c "$B_SUCCESSES") + (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_C_ID" assert_kind_count recv_from_b "$B_SUCCESSES") +fi + +if [[ "$TOTAL_FAILURES" -ne 0 ]]; then + echo + echo "At least one client call failed. Sample failure logs:" + find "$TMP_DIR" -name '*-client-*-seq-*.log' -type f -print0 \ + | xargs -0 grep -l "Error\|failed\|panic" \ + | head -n 10 \ + | while read -r log_file; do + echo "--- $log_file ---" + cat "$log_file" + done + exit 1 +fi + +echo +echo "Run complete." +echo "Flows exercised independently:" +echo "A reducer calls B" +echo "B reducer calls C" +echo "Use these database identities to inspect state manually if needed:" +echo "A: $DB_A_ID" +echo "B: $DB_B_ID" +echo "C: $DB_C_ID" diff --git a/modules/chain-call-repro/src/lib.rs b/modules/chain-call-repro/src/lib.rs new file mode 100644 index 00000000000..14719da347c --- /dev/null +++ b/modules/chain-call-repro/src/lib.rs @@ -0,0 +1,118 @@ +use spacetimedb::{Identity, ReducerContext, SpacetimeType, Table}; + +#[derive(SpacetimeType, Clone)] +pub struct CallPayload { + pub client_label: String, + pub seq: u64, + pub message: String, +} + +#[spacetimedb::table(accessor = call_log, public)] +pub struct CallLog { + #[primary_key] + #[auto_inc] + id: u64, + kind: String, + client_label: String, + seq: u64, + message: String, +} + +fn log_entry(ctx: &ReducerContext, kind: &str, payload: &CallPayload) { + ctx.db.call_log().insert(CallLog { + id: 0, + kind: kind.to_string(), + client_label: payload.client_label.clone(), + seq: payload.seq, + message: payload.message.clone(), + }); +} + +fn burn(iters: u64) { + if iters == 0 { + return; + } + + let mut x = 1u64; + for i in 0..iters { + x = x.wrapping_mul(6364136223846793005u64).wrapping_add(i | 1); + } + if x == 0 { + panic!("impossible burn result"); + } +} + +#[spacetimedb::reducer] +pub fn record_on_b(ctx: &ReducerContext, payload: CallPayload, burn_iters: u64) -> Result<(), String> { + burn(burn_iters); + log_entry(ctx, "recv_from_a", &payload); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn record_on_c(ctx: &ReducerContext, payload: CallPayload, burn_iters: u64) -> Result<(), String> { + burn(burn_iters); + log_entry(ctx, "recv_from_b", &payload); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn call_b_from_a( + ctx: &ReducerContext, + b_hex: String, + client_label: String, + seq: u64, + message: String, + burn_iters: u64, +) -> Result<(), String> { + burn(burn_iters); + + let b = Identity::from_hex(&b_hex).expect("invalid B identity"); + let payload = CallPayload { + client_label, + seq, + message, + }; + let args = spacetimedb::spacetimedb_lib::bsatn::to_vec(&(payload.clone(), burn_iters)) + .expect("failed to encode args for record_on_b"); + spacetimedb::remote_reducer::call_reducer_on_db_2pc(b, "record_on_b", &args) + .map_err(|e| format!("call_b_from_a: call to B failed: {e}"))?; + + log_entry(ctx, "sent_to_b", &payload); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn call_c_from_b( + ctx: &ReducerContext, + c_hex: String, + client_label: String, + seq: u64, + message: String, + burn_iters: u64, +) -> Result<(), String> { + burn(burn_iters); + + let c = Identity::from_hex(&c_hex).expect("invalid C identity"); + let payload = CallPayload { + client_label, + seq, + message, + }; + let args = spacetimedb::spacetimedb_lib::bsatn::to_vec(&(payload.clone(), burn_iters)) + .expect("failed to encode args for record_on_c"); + spacetimedb::remote_reducer::call_reducer_on_db_2pc(c, "record_on_c", &args) + .map_err(|e| format!("call_c_from_b: call to C failed: {e}"))?; + + log_entry(ctx, "sent_to_c", &payload); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn assert_kind_count(ctx: &ReducerContext, kind: String, expected: u64) -> Result<(), String> { + let actual = ctx.db.call_log().iter().filter(|row| row.kind == kind).count() as u64; + if actual != expected { + return Err(format!("expected kind '{kind}' count {expected}, got {actual}")); + } + Ok(()) +} From 12e5d50c7d473bbc9383d9e7edebf7a0a2ce4230 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Tue, 31 Mar 2026 09:40:02 -0700 Subject: [PATCH 25/33] make a disarm more solid --- crates/core/src/host/global_tx.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index 84e3fb17b49..b8a4571e879 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -177,17 +177,21 @@ impl<'a> WaitRegistration<'a> { self.wait_id.expect("registered waiter must still have a wait id") } - fn disarm(mut self) { - self.wait_id = None; + fn disarm(mut self, ls: &mut std::sync::MutexGuard<'_, LockState>) { + self.remove_waiter(ls); + } + + fn remove_waiter(&mut self, ls: &mut LockState) { + if let Some(wait_id) = self.wait_id.take() { + self.manager.remove_waiter_by_id(ls, wait_id); + } } } impl Drop for WaitRegistration<'_> { fn drop(&mut self) { - if let Some(wait_id) = self.wait_id.take() { - let mut ls = self.manager.lock_state.lock().unwrap(); - self.manager.remove_waiter_by_id(&mut ls, wait_id); - } + let mut ls = self.manager.lock_state.lock().unwrap(); + self.remove_waiter(&mut ls); } } @@ -345,7 +349,7 @@ impl GlobalTxManager { state.owner = Some(tx_id); self.remove_waiter_locked(&mut state, &tx_id); if let Some(registration) = registration.take() { - registration.disarm(); + registration.disarm(&mut state); } return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } @@ -366,9 +370,10 @@ impl GlobalTxManager { (notify, None, new_registration) } Some(owner) if owner == tx_id => { + log::warn!("global transaction {tx_id} is trying to acquire the lock it already holds. This should not happen and may indicate a bug in the caller's logic, but we'll allow it to proceed without deadlocking on itself."); self.remove_waiter_locked(&mut state, &tx_id); if let Some(registration) = registration.take() { - registration.disarm(); + registration.disarm(&mut state); } return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } From 8ba6d0cd72d24f12a5aeece91ff192dbb31c0313 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Tue, 31 Mar 2026 11:24:02 -0700 Subject: [PATCH 26/33] Fix an internal deadlock --- crates/core/src/host/global_tx.rs | 21 ++++++++++++++++++++- crates/core/src/host/module_host.rs | 21 ++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index b8a4571e879..56444ab1a38 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -179,6 +179,7 @@ impl<'a> WaitRegistration<'a> { fn disarm(mut self, ls: &mut std::sync::MutexGuard<'_, LockState>) { self.remove_waiter(ls); + self.wait_id = None; } fn remove_waiter(&mut self, ls: &mut LockState) { @@ -190,6 +191,9 @@ impl<'a> WaitRegistration<'a> { impl Drop for WaitRegistration<'_> { fn drop(&mut self) { + if self.wait_id.is_none() { + return; + } let mut ls = self.manager.lock_state.lock().unwrap(); self.remove_waiter(&mut ls); } @@ -208,6 +212,7 @@ impl<'a> GlobalTxLockGuard<'a> { } pub fn disarm(mut self) { + log::warn!("Disarming a lock guard without releasing the lock for tx_id {}", self.tx_id()); self.tx_id = None; } } @@ -346,11 +351,13 @@ impl GlobalTxManager { let mut state = self.lock_state.lock().unwrap(); match state.owner { None if self.is_next_waiter_locked(&state, tx_id) => { + log::info!("setting owner to {tx_id}"); state.owner = Some(tx_id); self.remove_waiter_locked(&mut state, &tx_id); if let Some(registration) = registration.take() { registration.disarm(&mut state); } + log::info!("global transaction {tx_id} acquired the lock"); return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } None => { @@ -366,6 +373,8 @@ impl GlobalTxManager { } else { self.ensure_waiter_locked(&mut state, tx_id) }; + + log::info!("global transaction {tx_id} is waiting for the lock; no current owner"); let new_registration = registration.is_none().then(|| WaitRegistration::new(self, wait_id)); (notify, None, new_registration) } @@ -447,7 +456,11 @@ impl GlobalTxManager { return AcquireDisposition::Cancelled; } } - _ = notify.notified() => {} + _ = notify.notified() => { + log::info!( + "global transaction {tx_id} was notified of a potential lock availability change; re-checking lock state" + ); + } } } } @@ -455,6 +468,7 @@ impl GlobalTxManager { pub fn release(&self, tx_id: &GlobalTxId) { let mut state = self.lock_state.lock().unwrap(); if state.owner.as_ref() == Some(tx_id) { + log::info!("Releasing lock for tx_id {}", tx_id); state.owner = None; state.wounded_owners.remove(tx_id); self.notify_next_waiter_locked(&state); @@ -501,6 +515,7 @@ impl GlobalTxManager { if let Some(wait_key) = state.waiting.first() && let Some(wait_entry) = state.wait_entries.get(&wait_key.wait_id) { + log::info!("Notifying next waiter for tx_id {}", wait_entry.tx_id); wait_entry.notify.notify_one(); } } @@ -513,8 +528,10 @@ impl GlobalTxManager { } fn remove_waiter_by_id(&self, state: &mut LockState, wait_id: u64) { + log::info!("Removing waiter with wait_id {}", wait_id); let was_head = state.waiting.first().map(|w| w.wait_id) == Some(wait_id); if let Some(wait_entry) = state.wait_entries.remove(&wait_id) { + log::info!("Removing waiter with wait_id {}, tx_id {}", wait_id, wait_entry.tx_id); state.waiter_ids_by_tx.remove(&wait_entry.tx_id); state.waiting.remove(&WaitKey { tx_id: wait_entry.tx_id, @@ -523,6 +540,8 @@ impl GlobalTxManager { if was_head && state.owner.is_none() { self.notify_next_waiter_locked(state); } + } else { + log::warn!("Trying to remove non-existent waiter with wait_id {}, current_owner: {:?}", wait_id, state.owner); } } diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 4424899f3ec..cf54e47de25 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1840,6 +1840,10 @@ impl ModuleHost { // recovery. Falls back to `caller_identity` when `None` (e.g., internal calls). coordinator_identity_override: Option, ) -> Result<(String, ReducerCallResult, Option), ReducerCallError> { + if tx_id.is_none() { + log::error!("prepare_reducer called without tx_id: caller_identity={caller_identity}, reducer_name={reducer_name}"); + } + let tx_id = tx_id.ok_or(ReducerCallError::NoSuchReducer)?; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::OnceLock; // Counter seeded from current time on first use so that restarts begin from a @@ -1875,7 +1879,7 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, - tx_id, + tx_id: Some(tx_id), client: None, request_id: None, timer: None, @@ -1889,6 +1893,8 @@ impl ModuleHost { // Include the coordinator identity so prepare_ids from different coordinators // cannot collide on the participant's st_2pc_state table. + let prepare_id = tx_id.to_string(); + /* let prepare_tx_component = tx_id .map(|tx_id| tx_id.to_string()) .unwrap_or_else(|| format!("legacy:{}:00000000", caller_identity.to_hex())); @@ -1897,6 +1903,7 @@ impl ModuleHost { prepare_tx_component, PREPARE_COUNTER.fetch_add(1, Ordering::Relaxed), ); + */ // Channel for signalling PREPARED result back to this task. let (prepared_tx, prepared_rx) = tokio::sync::oneshot::channel::<(ReducerCallResult, Option)>(); @@ -1909,7 +1916,7 @@ impl ModuleHost { decision_sender: decision_tx, }, ); - if let Some(tx_id) = tx_id { + //if let Some(tx_id) = tx_id { let session = self.replica_ctx().global_tx_manager.ensure_session( tx_id, super::global_tx::GlobalTxRole::Participant, @@ -1942,7 +1949,7 @@ impl ModuleHost { )); } } - } + //} // Spawn a background task that runs the reducer and holds the write lock // until we send a decision. The executor thread blocks inside @@ -1969,23 +1976,23 @@ impl ModuleHost { match prepared_rx.await { Ok((result, return_value)) => { if matches!(result.outcome, ReducerOutcome::Committed) { - if let Some(tx_id) = tx_id { + //if let Some(tx_id) = tx_id { self.replica_ctx() .global_tx_manager .mark_state(&tx_id, super::global_tx::GlobalTxState::Prepared); - } + // } Ok((prepare_id, result, return_value)) } else { // Reducer failed — remove the entry we registered (no hold in progress). self.prepared_txs.remove(&prepare_id); - if let Some(tx_id) = tx_id { + // if let Some(tx_id) = tx_id { self.replica_ctx().global_tx_manager.remove_prepare_mapping(&prepare_id); self.replica_ctx() .global_tx_manager .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborted); self.replica_ctx().global_tx_manager.release(&tx_id); self.replica_ctx().global_tx_manager.remove_session(&tx_id); - } + // } Ok((String::new(), result, return_value)) } } From 1ad310c918b254089d20d9dd6f9b6b91b8129811 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Tue, 31 Mar 2026 14:12:44 -0700 Subject: [PATCH 27/33] Various fixes --- crates/client-api/src/routes/database.rs | 104 +++++++--- crates/core/src/host/global_tx.rs | 60 ++++-- crates/core/src/host/instance_env.rs | 33 ++-- crates/core/src/host/module_host.rs | 128 +++++++++--- .../src/host/wasm_common/module_host_actor.rs | 23 +-- .../src/host/wasmtime/wasm_instance_env.rs | 23 +-- .../core/src/host/wasmtime/wasmtime_module.rs | 2 +- modules/chain-call-repro/run.sh | 183 +++++++++++++++--- modules/chain-call-repro/src/lib.rs | 10 + 9 files changed, 420 insertions(+), 146 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index bfe699dcfe7..746652dad82 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -46,6 +46,7 @@ use spacetimedb_lib::{sats, AlgebraicValue, GlobalTxId, Hash, ProductValue, Time use spacetimedb_schema::auto_migrate::{ MigrationPolicy as SchemaMigrationPolicy, MigrationToken, PrettyPrintStyle as AutoMigratePrettyPrintStyle, }; +use tokio::time::sleep; use super::subscribe::{handle_websocket, HasWebSocketOptions}; @@ -89,6 +90,7 @@ pub struct CallParams { pub const NO_SUCH_DATABASE: (StatusCode, &str) = (StatusCode::NOT_FOUND, "No such database."); const MISDIRECTED: (StatusCode, &str) = (StatusCode::NOT_FOUND, "Database is not scheduled on this host"); +const PREPARE_ID_HEADER: &str = "X-Prepare-Id"; fn map_reducer_error(e: ReducerCallError, reducer: &str) -> (StatusCode, String) { let status_code = match e { @@ -96,6 +98,7 @@ fn map_reducer_error(e: ReducerCallError, reducer: &str) -> (StatusCode, String) log::debug!("Attempt to call reducer {reducer} with invalid arguments"); StatusCode::BAD_REQUEST } + ReducerCallError::InvalidPrepareId(_) => StatusCode::BAD_REQUEST, ReducerCallError::NoSuchModule(_) | ReducerCallError::ScheduleReducerNotFound => StatusCode::NOT_FOUND, ReducerCallError::NoSuchReducer => { log::debug!("Attempt to call non-existent reducer {reducer}"); @@ -143,7 +146,7 @@ pub async fn call( body: Bytes, ) -> axum::response::Result { let caller_identity = auth.claims.identity; - let tx_id = headers + let requested_tx_id = headers .get(TX_ID_HEADER) .and_then(|v| v.to_str().ok()) .and_then(|s| s.parse::().ok()); @@ -155,6 +158,9 @@ pub async fn call( let connection_id = generate_random_connection_id(); let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; + let tx_id = requested_tx_id.unwrap_or_else(|| module.mint_global_tx_id(Timestamp::now())); + const MAX_WOUNDED_RETRIES: usize = 10; + const MAX_WOUNDED_BACKOFF: Duration = Duration::from_millis(1000); // Call the database's `client_connected` reducer, if any. // If it fails or rejects the connection, bail. @@ -164,35 +170,56 @@ pub async fn call( .map_err(client_connected_error_to_response)?; let mut reducer_return_value: Option = None; - let result = match module - .call_reducer_with_return( - caller_identity, - Some(connection_id), - tx_id, - None, - None, - None, - &reducer, - args.clone(), - ) - .await - { - Ok((rcr, return_value)) => { - reducer_return_value = return_value; - Ok(CallResult::Reducer(rcr)) - } - Err(ReducerCallError::NoSuchReducer | ReducerCallError::ScheduleReducerNotFound) => { - // Not a reducer — try procedure instead - match module - .call_procedure(caller_identity, Some(connection_id), None, &reducer, args) - .await - .result - { - Ok(res) => Ok(CallResult::Procedure(res)), - Err(e) => Err(map_procedure_error(e, &reducer)), + let mut tx_id = tx_id; + let mut wound_backoff = Duration::from_millis(10); + let result = loop { + match module + .call_reducer_with_return( + caller_identity, + Some(connection_id), + Some(tx_id), + None, + None, + None, + &reducer, + args.clone(), + ) + .await + { + Ok((rcr, return_value)) => { + if matches!(rcr.outcome, ReducerOutcome::Wounded(_)) { + if tx_id.attempt >= MAX_WOUNDED_RETRIES as u32 { + log::warn!("HTTP reducer call was wounded on final attempt. Returning error to caller."); + reducer_return_value = return_value; + break Ok(CallResult::Reducer(rcr)); + } + log::info!( + "HTTP reducer call was wounded on attempt {}, retrying after {:?} with new transaction ID {}", + tx_id.attempt, + wound_backoff, + tx_id + ); + sleep(wound_backoff).await; + wound_backoff = wound_backoff.mul_f32(2.0).min(MAX_WOUNDED_BACKOFF); + tx_id = tx_id.next_attempt(); + continue; + } + reducer_return_value = return_value; + break Ok(CallResult::Reducer(rcr)); + } + Err(ReducerCallError::NoSuchReducer | ReducerCallError::ScheduleReducerNotFound) => { + // Not a reducer — try procedure instead + break match module + .call_procedure(caller_identity, Some(connection_id), None, &reducer, args) + .await + .result + { + Ok(res) => Ok(CallResult::Procedure(res)), + Err(e) => Err(map_procedure_error(e, &reducer)), + }; } + Err(e) => break Err(map_reducer_error(e, &reducer)), } - Err(e) => Err(map_reducer_error(e, &reducer)), }; module @@ -288,6 +315,17 @@ pub async fn prepare( .get("X-Coordinator-Identity") .and_then(|v| v.to_str().ok()) .and_then(|s| spacetimedb_lib::Identity::from_hex(s).ok()); + let supplied_prepare_id = headers + .get(PREPARE_ID_HEADER) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + if tx_id.is_some() && supplied_prepare_id.is_none() { + return Err(( + StatusCode::BAD_REQUEST, + format!("missing required {PREPARE_ID_HEADER} header for 2PC prepare request"), + ) + .into()); + } let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; @@ -295,7 +333,15 @@ pub async fn prepare( // call_identity_connected/disconnected submit jobs to the module's executor, which // will be blocked holding the 2PC write lock after prepare_reducer returns — deadlock. let result = module - .prepare_reducer(caller_identity, None, tx_id, &reducer, args.0, coordinator_identity) + .prepare_reducer( + caller_identity, + None, + tx_id, + &reducer, + args.0, + coordinator_identity, + supplied_prepare_id, + ) .await; match result { diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index 56444ab1a38..4036d03e27d 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -36,7 +36,7 @@ pub struct GlobalTxSession { wounded_tx: watch::Sender, state: Mutex, prepare_id: Mutex>, - participants: Mutex>, + participants: Mutex>, } impl GlobalTxSession { @@ -50,7 +50,7 @@ impl GlobalTxSession { wounded_tx, state: Mutex::new(GlobalTxState::Running), prepare_id: Mutex::new(None), - participants: Mutex::new(HashMap::new()), + participants: Mutex::new(Vec::new()), } } @@ -87,16 +87,11 @@ impl GlobalTxSession { } pub fn add_participant(&self, db_identity: Identity, prepare_id: String) { - self.participants.lock().unwrap().insert(db_identity, prepare_id); + self.participants.lock().unwrap().push((db_identity, prepare_id)); } pub fn participants(&self) -> Vec<(Identity, String)> { - self.participants - .lock() - .unwrap() - .iter() - .map(|(db, pid)| (*db, pid.clone())) - .collect() + self.participants.lock().unwrap().clone() } } @@ -347,8 +342,11 @@ impl GlobalTxManager { return AcquireDisposition::Cancelled; } - let (notify, owner_to_wound, new_registration) = { + let (notify, owner_to_wound, new_registration): (Arc, Option, Option>) = { let mut state = self.lock_state.lock().unwrap(); + if state.owner.is_none() { + self.prune_stale_head_waiters_locked(&mut state); + } match state.owner { None if self.is_next_waiter_locked(&state, tx_id) => { log::info!("setting owner to {tx_id}"); @@ -373,8 +371,20 @@ impl GlobalTxManager { } else { self.ensure_waiter_locked(&mut state, tx_id) }; + let head_waiter = state.waiting.first().map(|wait_key| wait_key.tx_id); + if let Some(head_waiter) = head_waiter + && head_waiter != tx_id + { + log::info!( + "global transaction {tx_id} observed ownerless lock while queued behind head waiter {head_waiter}; nudging head waiter" + ); + self.notify_next_waiter_locked(&state); + } - log::info!("global transaction {tx_id} is waiting for the lock; no current owner"); + log::info!( + "global transaction {tx_id} is waiting for the lock; no current owner; head waiter: {:?}", + head_waiter + ); let new_registration = registration.is_none().then(|| WaitRegistration::new(self, wait_id)); (notify, None, new_registration) } @@ -550,6 +560,34 @@ impl GlobalTxManager { .map(|session| !(session.role == GlobalTxRole::Participant && session.state() == GlobalTxState::Prepared)) .unwrap_or(true) } + + fn prune_stale_head_waiters_locked(&self, state: &mut LockState) { + while let Some(wait_key) = state.waiting.first().copied() { + let tx_id = wait_key.tx_id; + let Some(session) = self.get_session(&tx_id) else { + log::warn!( + "pruning stale head waiter {tx_id}: no session exists while owner is None" + ); + self.remove_waiter_by_id(state, wait_key.wait_id); + continue; + }; + let session_state = session.state(); + let wounded = session.is_wounded(); + let terminalish = wounded + || matches!( + session_state, + GlobalTxState::Committed | GlobalTxState::Aborted | GlobalTxState::Aborting + ); + if terminalish { + log::warn!( + "pruning stale head waiter {tx_id}: state={session_state:?} wounded={wounded} while owner is None" + ); + self.remove_waiter_by_id(state, wait_key.wait_id); + continue; + } + break; + } + } } #[cfg(test)] diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 7be2ee47ad0..1ec455d1371 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -39,6 +39,8 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use std::vec::IntoIter; +const PREPARE_ID_HEADER: &str = "X-Prepare-Id"; + pub struct InstanceEnv { pub replica_ctx: Arc, pub scheduler: Scheduler, @@ -57,11 +59,13 @@ pub struct InstanceEnv { in_anon_tx: bool, /// A procedure's last known transaction offset. procedure_last_tx_offset: Option, - /// 2PC: prepared participants from `call_reducer_on_db_2pc` calls. + /// 2PC: participant prepare targets from `call_reducer_on_db_2pc` calls. /// Each entry is (database_identity, prepare_id). /// After the coordinator's reducer commits, these are committed; - /// on failure, they are aborted. - pub prepared_participants: Vec<(Identity, String)>, + /// on failure, they are aborted. Entries are registered before the + /// HTTP prepare request is sent so wound-driven abort fanout can target + /// in-flight prepares using coordinator-assigned ids. + pub contacted_participants: Vec<(Identity, String)>, } /// `InstanceEnv` needs to be `Send` because it is created on the host thread @@ -247,7 +251,7 @@ impl InstanceEnv { current_tx_id: None, in_anon_tx: false, procedure_last_tx_offset: None, - prepared_participants: Vec::new(), + contacted_participants: Vec::new(), } } @@ -1093,8 +1097,9 @@ impl InstanceEnv { /// Call a reducer on a remote database using the 2PC prepare protocol. /// /// Like [`Self::call_reducer_on_db`], but POSTs to `/prepare/{reducer}` instead of - /// `/call/{reducer}`. On success, parses the `X-Prepare-Id` response header and stores - /// `(database_identity, prepare_id)` in [`Self::prepared_participants`]. + /// `/call/{reducer}`. The coordinator generates the `prepare_id` up front, sends it + /// in the request, and stores `(database_identity, prepare_id)` in + /// [`Self::contacted_participants`] before the HTTP round-trip begins. /// /// Blocks the calling thread for the duration of the HTTP round-trip. /// @@ -1104,7 +1109,7 @@ impl InstanceEnv { database_identity: Identity, reducer_name: &str, args: bytes::Bytes, - ) -> Result<(u16, bytes::Bytes, Option), NodesError> { + ) -> Result<(u16, bytes::Bytes, String), NodesError> { let caller_identity = self.replica_ctx.database.database_identity; let tx_id = self.current_tx_id().ok_or_else(|| { NodesError::HttpError("2PC remote reducer call requires an active distributed transaction id".to_owned()) @@ -1119,6 +1124,10 @@ impl InstanceEnv { .global_tx_manager .ensure_session(tx_id, GlobalTxRole::Coordinator, tx_id.creator_db); session.set_state(GlobalTxState::Preparing); + let prepare_id = super::module_host::generate_prepare_id(tx_id, caller_identity); + session.add_participant(database_identity, prepare_id.clone()); + self.contacted_participants + .push((database_identity, prepare_id.clone())); let start = Instant::now(); @@ -1139,6 +1148,7 @@ impl InstanceEnv { .post(&url) .header(http::header::CONTENT_TYPE, "application/octet-stream") .header("X-Coordinator-Identity", caller_identity.to_hex().to_string()) + .header(PREPARE_ID_HEADER, prepare_id.clone()) .header(TX_ID_HEADER, tx_id.to_string()) .body(args.to_vec()); if let Some(ref token) = self.replica_ctx.call_reducer_auth_token { @@ -1159,13 +1169,8 @@ impl InstanceEnv { move || manager.is_wounded(&tx_id), |resp| { let status = resp.status().as_u16(); - let prepare_id = resp - .headers() - .get("X-Prepare-Id") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_owned()); let body = resp.bytes()?; - Ok((status, body, prepare_id)) + Ok((status, body)) }, ); let result = match outcome { @@ -1182,7 +1187,7 @@ impl InstanceEnv { .with_label_values(&caller_identity) .observe(start.elapsed().as_secs_f64()); - result + result.map(|(status, body)| (status, body, prepare_id)) } } diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index cf54e47de25..6b8ffa49f12 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -68,7 +68,8 @@ use spacetimedb_schema::schema::{Schema, TableSchema}; use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; use std::fmt; -use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::OnceLock; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; use tokio::sync::oneshot; @@ -973,6 +974,27 @@ pub enum ReducerCallError { ScheduleReducerNotFound, #[error("can't directly call special {0:?} lifecycle reducer")] LifecycleReducer(Lifecycle), + #[error("invalid prepare id: {0}")] + InvalidPrepareId(String), +} + +static PREPARE_COUNTER: AtomicU64 = AtomicU64::new(0); +static PREPARE_COUNTER_INIT: OnceLock<()> = OnceLock::new(); + +fn next_prepare_counter() -> u64 { + PREPARE_COUNTER_INIT.get_or_init(|| { + let seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_micros() as u64; + PREPARE_COUNTER.store(seed, Ordering::Relaxed); + }); + PREPARE_COUNTER.fetch_add(1, Ordering::Relaxed) +} + +pub(crate) fn generate_prepare_id(tx_id: GlobalTxId, coordinator_identity: Identity) -> String { + let counter = next_prepare_counter(); + format!("prepare-{tx_id}-{counter:016x}-{}", coordinator_identity.to_hex()) } #[derive(Debug, PartialEq, Eq)] @@ -1839,25 +1861,12 @@ impl ModuleHost { // When `Some`, used for `prepare_id` namespacing and stored in `st_2pc_state` for // recovery. Falls back to `caller_identity` when `None` (e.g., internal calls). coordinator_identity_override: Option, + supplied_prepare_id: Option, ) -> Result<(String, ReducerCallResult, Option), ReducerCallError> { if tx_id.is_none() { log::error!("prepare_reducer called without tx_id: caller_identity={caller_identity}, reducer_name={reducer_name}"); } let tx_id = tx_id.ok_or(ReducerCallError::NoSuchReducer)?; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::OnceLock; - // Counter seeded from current time on first use so that restarts begin from a - // different value than any existing st_2pc_state entries (which hold IDs from - // previous sessions starting at much smaller counter values). - static PREPARE_COUNTER: AtomicU64 = AtomicU64::new(0); - static PREPARE_COUNTER_INIT: OnceLock<()> = OnceLock::new(); - PREPARE_COUNTER_INIT.get_or_init(|| { - let seed = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_micros() as u64; - PREPARE_COUNTER.store(seed, Ordering::Relaxed); - }); let (reducer_id, reducer_def) = self .info @@ -1890,20 +1899,15 @@ impl ModuleHost { // Resolve the effective coordinator identity before generating the prepare_id so // the prefix is namespaced correctly even when called from the HTTP prepare handler. let coordinator_identity = coordinator_identity_override.unwrap_or(caller_identity); - - // Include the coordinator identity so prepare_ids from different coordinators - // cannot collide on the participant's st_2pc_state table. - let prepare_id = tx_id.to_string(); - /* - let prepare_tx_component = tx_id - .map(|tx_id| tx_id.to_string()) - .unwrap_or_else(|| format!("legacy:{}:00000000", caller_identity.to_hex())); - let prepare_id = format!( - "prepare-{}-{}", - prepare_tx_component, - PREPARE_COUNTER.fetch_add(1, Ordering::Relaxed), - ); - */ + let prepare_id = supplied_prepare_id.unwrap_or_else(|| generate_prepare_id(tx_id, coordinator_identity)); + let prepare_tx_id = Self::tx_id_from_prepare_id(&prepare_id).ok_or_else(|| { + ReducerCallError::InvalidPrepareId(format!("prepare_id '{prepare_id}' is not parseable")) + })?; + if prepare_tx_id != tx_id { + return Err(ReducerCallError::InvalidPrepareId(format!( + "prepare_id '{prepare_id}' encodes tx_id {prepare_tx_id}, expected {tx_id}" + ))); + } // Channel for signalling PREPARED result back to this task. let (prepared_tx, prepared_rx) = tokio::sync::oneshot::channel::<(ReducerCallResult, Option)>(); @@ -2188,7 +2192,10 @@ impl ModuleHost { fn tx_id_from_prepare_id(prepare_id: &str) -> Option { let raw = prepare_id.strip_prefix("prepare-")?; - let (tx_component, _) = raw.rsplit_once('-')?; + let mut parts = raw.rsplitn(3, '-'); + let _tail = parts.next()?; + let middle = parts.next()?; + let tx_component = parts.next().unwrap_or(middle); if tx_component.starts_with("legacy:") { return None; } @@ -2413,6 +2420,7 @@ impl ModuleHost { &row.reducer_name, args, Some(coordinator_identity), + None, ) .await { @@ -3365,6 +3373,10 @@ impl ModuleHost { self.replica_ctx().relational_db() } + pub fn mint_global_tx_id(&self, start_ts: Timestamp) -> GlobalTxId { + self.replica_ctx().mint_global_tx_id(start_ts) + } + pub(crate) fn replica_ctx(&self) -> &ReplicaContext { match &*self.inner { ModuleHostInner::Wasm(wasm) => wasm.instance_manager.module.replica_ctx(), @@ -3408,16 +3420,17 @@ fn args_error_log_message(function_kind: &str, function_name: &str) -> String { #[cfg(test)] mod tests { - use super::ModuleHost; + use super::{generate_prepare_id, ModuleHost}; use crate::client::{ ClientActorId, ClientConfig, ClientConnectionReceiver, ClientConnectionSender, OutboundMessage, Protocol, WsVersion, }; use crate::db::relational_db::tests_utils::{insert, with_auto_commit, TestDB}; + use crate::host::global_tx::{GlobalTxManager, GlobalTxRole}; use crate::subscription::module_subscription_actor::ModuleSubscriptions; use spacetimedb_client_api_messages::websocket::{common::RowListLen as _, v1 as ws_v1, v2 as ws_v2}; use spacetimedb_lib::identity::AuthCtx; - use spacetimedb_lib::{AlgebraicType, Identity}; + use spacetimedb_lib::{AlgebraicType, GlobalTxId, Identity, Timestamp}; use spacetimedb_sats::product; use std::sync::Arc; @@ -3514,4 +3527,55 @@ mod tests { Ok(()) } + + #[test] + fn generated_prepare_id_round_trips_tx_id() { + let tx_id = GlobalTxId { + start_ts: Timestamp::from_micros_since_unix_epoch(123), + creator_db: Identity::from_byte_array([7; 32]), + nonce: 9, + attempt: 2, + }; + let prepare_id = generate_prepare_id(tx_id, Identity::from_byte_array([3; 32])); + assert_eq!(ModuleHost::tx_id_from_prepare_id(&prepare_id), Some(tx_id)); + } + + #[test] + fn generated_prepare_ids_are_unique_per_call() { + let tx_id = GlobalTxId { + start_ts: Timestamp::from_micros_since_unix_epoch(456), + creator_db: Identity::from_byte_array([8; 32]), + nonce: 10, + attempt: 3, + }; + let coordinator = Identity::from_byte_array([4; 32]); + let a = generate_prepare_id(tx_id, coordinator); + let b = generate_prepare_id(tx_id, coordinator); + assert_ne!(a, b); + assert_eq!(ModuleHost::tx_id_from_prepare_id(&a), Some(tx_id)); + assert_eq!(ModuleHost::tx_id_from_prepare_id(&b), Some(tx_id)); + } + + #[test] + fn global_tx_session_keeps_multiple_prepare_ids_for_same_participant() { + let tx_id = GlobalTxId { + start_ts: Timestamp::from_micros_since_unix_epoch(789), + creator_db: Identity::from_byte_array([9; 32]), + nonce: 11, + attempt: 4, + }; + let participant = Identity::from_byte_array([5; 32]); + let manager = GlobalTxManager::default(); + let session = manager.ensure_session(tx_id, GlobalTxRole::Coordinator, Identity::from_byte_array([6; 32])); + session.add_participant(participant, "prepare-a".to_string()); + session.add_participant(participant, "prepare-b".to_string()); + + assert_eq!( + session.participants(), + vec![ + (participant, "prepare-a".to_string()), + (participant, "prepare-b".to_string()) + ] + ); + } } diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 26d524f0f21..2af85569dcc 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1211,12 +1211,12 @@ impl InstanceCommon { inst: &mut I, ) -> (ReducerCallResult, Option, bool) { let managed_global_tx_id = if tx.is_none() { params.tx_id } else { None }; - let (mut tx, mut event, client, trapped) = self.run_reducer_no_commit(tx, params, inst); + let (mut tx, event, client, trapped) = self.run_reducer_no_commit(tx, params, inst); let energy_quanta_used = event.energy_quanta_used; let total_duration = event.host_execution_duration; - // Take participants before commit so we can write the coordinator log atomically. + // Take participant prepare targets before commit so we can write the coordinator log atomically. let prepared_participants = inst.take_prepared_participants(); // If this coordinator tx is committed and has participants, write coordinator log @@ -1231,13 +1231,6 @@ impl InstanceCommon { } } - if let Some(status) = check_wounded(inst.replica_ctx(), managed_global_tx_id) - && matches!(event.status, EventStatus::Committed(_)) - { - event.status = status; - event.reducer_return_value = None; - } - let commit_result = commit_and_broadcast_event(&self.info.subscriptions, client, event, tx); let commit_tx_offset = commit_result.tx_offset; let event = commit_result.event; @@ -1313,10 +1306,18 @@ impl InstanceCommon { } } Ok(status) => { - log::error!("2PC {action}: failed for {prepare_id} on {db_identity}: status {status}"); + if committed { + log::error!("2PC {action}: failed for {prepare_id} on {db_identity}: status {status}"); + } else { + log::warn!("2PC {action}: best-effort abort for {prepare_id} on {db_identity} returned status {status}"); + } } Err(e) => { - log::error!("2PC {action}: transport error for {prepare_id} on {db_identity}: {e}"); + if committed { + log::error!("2PC {action}: transport error for {prepare_id} on {db_identity}: {e}"); + } else { + log::warn!("2PC {action}: best-effort abort transport error for {prepare_id} on {db_identity}: {e}"); + } } } } diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 721db4215c0..8bd7e273359 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -2031,9 +2031,10 @@ impl WasmInstanceEnv { /// 2PC variant of `call_reducer_on_db`. /// /// Calls the remote database's `/prepare/{reducer}` endpoint instead of `/call/{reducer}`. - /// On success, parses the `X-Prepare-Id` header and stores the participant info in - /// `InstanceEnv::prepared_participants` so the runtime can commit/abort after the - /// coordinator's reducer completes. + /// The coordinator assigns the `prepare_id` before sending the request and stores the + /// participant info in `InstanceEnv::contacted_participants` so the runtime can + /// commit/abort after the coordinator's reducer completes, even if the request is + /// wounded while in flight. /// /// Returns the HTTP status code on success, writing the response body to `*out` /// as a [`BytesSource`]. @@ -2067,21 +2068,7 @@ impl WasmInstanceEnv { .call_reducer_on_db_2pc(database_identity, &reducer_name, args); match result { - Ok((status, body, prepare_id)) => { - // If we got a prepare_id, register this participant. - if let Some(pid) = prepare_id - && status < 300 - { - if let Some(tx_id) = env.instance_env.current_tx_id() { - let session = env.instance_env.replica_ctx.global_tx_manager.ensure_session( - tx_id, - crate::host::global_tx::GlobalTxRole::Coordinator, - tx_id.creator_db, - ); - session.add_participant(database_identity, pid.clone()); - } - env.instance_env.prepared_participants.push((database_identity, pid)); - } + Ok((status, body, _prepare_id)) => { let bytes_source = WasmInstanceEnv::create_bytes_source(env, body)?; bytes_source.0.write_to(mem, out)?; Ok(status as u32) diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 4d3b71b43d4..78424746d5b 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -577,7 +577,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { } fn take_prepared_participants(&mut self) -> Vec<(Identity, String)> { - core::mem::take(&mut self.store.data_mut().instance_env_mut().prepared_participants) + core::mem::take(&mut self.store.data_mut().instance_env_mut().contacted_participants) } #[tracing::instrument(level = "trace", skip_all)] diff --git a/modules/chain-call-repro/run.sh b/modules/chain-call-repro/run.sh index 454341436a2..80753472e13 100755 --- a/modules/chain-call-repro/run.sh +++ b/modules/chain-call-repro/run.sh @@ -8,13 +8,84 @@ A_CLIENTS="${A_CLIENTS:-4}" B_CLIENTS="${B_CLIENTS:-4}" ITERATIONS="${ITERATIONS:-25}" BURN_ITERS="${BURN_ITERS:-0}" +HOLD_ITERS="${HOLD_ITERS:-25000000}" RUN_ID="$(date +%Y%m%d%H%M%S)-$$" DB_A="independent-repro-a-${RUN_ID}" DB_B="independent-repro-b-${RUN_ID}" DB_C="independent-repro-c-${RUN_ID}" TMP_DIR="$(mktemp -d)" +PUBLISH_FIRST=1 +RUN_FOREVER=0 +RUN_A_CLIENTS=1 +RUN_B_CLIENTS=1 +DB_A_ID="${DB_A_ID:-}" +DB_B_ID="${DB_B_ID:-}" +DB_C_ID="${DB_C_ID:-}" + +usage() { + cat <<'EOF' +Usage: ./run.sh [options] + +Options: + --skip-publish Reuse existing DB identities from DB_A_ID, DB_B_ID, and DB_C_ID. + --forever Run client calls forever instead of stopping after ITERATIONS. + --only-a-client Run only A clients. + --only-b-client Run only B clients. + --help Show this help. + +Environment: + SPACETIME_SERVER Server name. Defaults to local. + A_CLIENTS Number of A clients. Defaults to 4. + B_CLIENTS Number of B clients. Defaults to 4. + ITERATIONS Calls per client when not using --forever. Defaults to 25. + BURN_ITERS Burn work per reducer call. Defaults to 0. + HOLD_ITERS Burn work after remote prepare succeeds. Defaults to 25000000. + DB_A_ID Required with --skip-publish. + DB_B_ID Required with --skip-publish. + DB_C_ID Required with --skip-publish. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-publish) + PUBLISH_FIRST=0 + ;; + --forever) + RUN_FOREVER=1 + ;; + --only-a-client) + RUN_B_CLIENTS=0 + ;; + --only-b-client) + RUN_A_CLIENTS=0 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + echo >&2 + usage >&2 + exit 1 + ;; + esac + shift +done + +if [[ "$RUN_A_CLIENTS" -eq 0 && "$RUN_B_CLIENTS" -eq 0 ]]; then + echo "nothing to do: both A and B clients were disabled" >&2 + exit 1 +fi cleanup() { + local pids + + pids="$(jobs -pr)" || true + if [[ -n "$pids" ]]; then + kill $pids 2>/dev/null || true + fi rm -rf "$TMP_DIR" } trap cleanup EXIT @@ -44,9 +115,12 @@ publish_db() { run_a_client() { local client_id="$1" local failures=0 + local log_file local seq - for ((seq = 1; seq <= ITERATIONS; seq++)); do + log_file="$TMP_DIR/a-client-${client_id}.log" + seq=1 + while :; do if ! ( cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_A_ID" call_b_from_a \ @@ -54,10 +128,16 @@ run_a_client() { "a-client-${client_id}" \ "$seq" \ "a-msg-client-${client_id}-seq-${seq}" \ - "$BURN_ITERS" - ) >"$TMP_DIR/a-client-${client_id}-seq-${seq}.log" 2>&1; then + "$BURN_ITERS" \ + "$HOLD_ITERS" + ) >"$log_file" 2>&1; then failures=$((failures + 1)) fi + + if [[ "$RUN_FOREVER" -eq 0 && "$seq" -ge "$ITERATIONS" ]]; then + break + fi + seq=$((seq + 1)) done printf '%s\n' "$failures" >"$TMP_DIR/a-client-${client_id}.failures" @@ -66,9 +146,12 @@ run_a_client() { run_b_client() { local client_id="$1" local failures=0 + local log_file local seq - for ((seq = 1; seq <= ITERATIONS; seq++)); do + log_file="$TMP_DIR/b-client-${client_id}.log" + seq=1 + while :; do if ! ( cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_B_ID" call_c_from_b \ @@ -76,47 +159,87 @@ run_b_client() { "b-client-${client_id}" \ "$seq" \ "b-msg-client-${client_id}-seq-${seq}" \ - "$BURN_ITERS" - ) >"$TMP_DIR/b-client-${client_id}-seq-${seq}.log" 2>&1; then + "$BURN_ITERS" \ + "$HOLD_ITERS" + ) >"$log_file" 2>&1; then failures=$((failures + 1)) fi + + if [[ "$RUN_FOREVER" -eq 0 && "$seq" -ge "$ITERATIONS" ]]; then + break + fi + seq=$((seq + 1)) done printf '%s\n' "$failures" >"$TMP_DIR/b-client-${client_id}.failures" } -echo "Publishing independent-call repro module to A, B, and C on server '$SERVER'..." -DB_C_ID="$(publish_db "$DB_C")" -DB_B_ID="$(publish_db "$DB_B")" -DB_A_ID="$(publish_db "$DB_A")" +if [[ "$PUBLISH_FIRST" -eq 1 ]]; then + echo "Publishing independent-call repro module to A, B, and C on server '$SERVER'..." + DB_C_ID="$(publish_db "$DB_C")" + DB_B_ID="$(publish_db "$DB_B")" + DB_A_ID="$(publish_db "$DB_A")" +else + if [[ -z "$DB_A_ID" || -z "$DB_B_ID" || -z "$DB_C_ID" ]]; then + echo "DB_A_ID, DB_B_ID, and DB_C_ID are required with --skip-publish" >&2 + exit 1 + fi +fi echo "A identity: $DB_A_ID" echo "B identity: $DB_B_ID" echo "C identity: $DB_C_ID" -echo "Starting $A_CLIENTS A-clients and $B_CLIENTS B-clients with $ITERATIONS calls each..." +echo "Client logs directory: $TMP_DIR" +if [[ "$RUN_A_CLIENTS" -eq 1 ]]; then + echo "A client logs: $TMP_DIR/a-client-.log" +fi +if [[ "$RUN_B_CLIENTS" -eq 1 ]]; then + echo "B client logs: $TMP_DIR/b-client-.log" +fi +if [[ "$RUN_FOREVER" -eq 1 ]]; then + echo "Starting clients in forever mode..." +else + echo "Starting clients with $ITERATIONS calls each..." +fi +echo "Prepare hold burn iters: $HOLD_ITERS" +echo "Workload note: run both A and B clients together to create contention on B and drive wound flow." +echo "A clients enabled: $RUN_A_CLIENTS ($A_CLIENTS configured)" +echo "B clients enabled: $RUN_B_CLIENTS ($B_CLIENTS configured)" -for ((client_id = 1; client_id <= A_CLIENTS; client_id++)); do - run_a_client "$client_id" & -done -for ((client_id = 1; client_id <= B_CLIENTS; client_id++)); do - run_b_client "$client_id" & -done +if [[ "$RUN_A_CLIENTS" -eq 1 ]]; then + for ((client_id = 1; client_id <= A_CLIENTS; client_id++)); do + run_a_client "$client_id" & + done +fi +if [[ "$RUN_B_CLIENTS" -eq 1 ]]; then + for ((client_id = 1; client_id <= B_CLIENTS; client_id++)); do + run_b_client "$client_id" & + done +fi wait +if [[ "$RUN_FOREVER" -eq 1 ]]; then + exit 0 +fi + A_FAILURES=0 -for ((client_id = 1; client_id <= A_CLIENTS; client_id++)); do - client_failures="$(cat "$TMP_DIR/a-client-${client_id}.failures")" - A_FAILURES=$((A_FAILURES + client_failures)) -done +if [[ "$RUN_A_CLIENTS" -eq 1 ]]; then + for ((client_id = 1; client_id <= A_CLIENTS; client_id++)); do + client_failures="$(cat "$TMP_DIR/a-client-${client_id}.failures")" + A_FAILURES=$((A_FAILURES + client_failures)) + done +fi B_FAILURES=0 -for ((client_id = 1; client_id <= B_CLIENTS; client_id++)); do - client_failures="$(cat "$TMP_DIR/b-client-${client_id}.failures")" - B_FAILURES=$((B_FAILURES + client_failures)) -done +if [[ "$RUN_B_CLIENTS" -eq 1 ]]; then + for ((client_id = 1; client_id <= B_CLIENTS; client_id++)); do + client_failures="$(cat "$TMP_DIR/b-client-${client_id}.failures")" + B_FAILURES=$((B_FAILURES + client_failures)) + done +fi -A_SUCCESSES=$((A_CLIENTS * ITERATIONS - A_FAILURES)) -B_SUCCESSES=$((B_CLIENTS * ITERATIONS - B_FAILURES)) +A_SUCCESSES=$((RUN_A_CLIENTS * A_CLIENTS * ITERATIONS - A_FAILURES)) +B_SUCCESSES=$((RUN_B_CLIENTS * B_CLIENTS * ITERATIONS - B_FAILURES)) TOTAL_FAILURES=$((A_FAILURES + B_FAILURES)) echo "Successful A->B calls: $A_SUCCESSES" @@ -124,12 +247,12 @@ echo "Failed A->B calls: $A_FAILURES" echo "Successful B->C calls: $B_SUCCESSES" echo "Failed B->C calls: $B_FAILURES" -if [[ "$A_SUCCESSES" -gt 0 ]]; then +if [[ "$RUN_A_CLIENTS" -eq 1 && "$A_SUCCESSES" -gt 0 ]]; then (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_A_ID" assert_kind_count sent_to_b "$A_SUCCESSES") (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_B_ID" assert_kind_count recv_from_a "$A_SUCCESSES") fi -if [[ "$B_SUCCESSES" -gt 0 ]]; then +if [[ "$RUN_B_CLIENTS" -eq 1 && "$B_SUCCESSES" -gt 0 ]]; then (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_B_ID" assert_kind_count sent_to_c "$B_SUCCESSES") (cd "$SCRIPT_DIR" && spacetime call --server "$SERVER" -- "$DB_C_ID" assert_kind_count recv_from_b "$B_SUCCESSES") fi @@ -137,7 +260,7 @@ fi if [[ "$TOTAL_FAILURES" -ne 0 ]]; then echo echo "At least one client call failed. Sample failure logs:" - find "$TMP_DIR" -name '*-client-*-seq-*.log' -type f -print0 \ + find "$TMP_DIR" -name '*-client-*.log' -type f -print0 \ | xargs -0 grep -l "Error\|failed\|panic" \ | head -n 10 \ | while read -r log_file; do diff --git a/modules/chain-call-repro/src/lib.rs b/modules/chain-call-repro/src/lib.rs index 14719da347c..8ff8047fd14 100644 --- a/modules/chain-call-repro/src/lib.rs +++ b/modules/chain-call-repro/src/lib.rs @@ -64,6 +64,7 @@ pub fn call_b_from_a( seq: u64, message: String, burn_iters: u64, + hold_iters: u64, ) -> Result<(), String> { burn(burn_iters); @@ -78,6 +79,10 @@ pub fn call_b_from_a( spacetimedb::remote_reducer::call_reducer_on_db_2pc(b, "record_on_b", &args) .map_err(|e| format!("call_b_from_a: call to B failed: {e}"))?; + // Hold A open after B is prepared so B keeps its global-tx admission lock + // long enough for concurrent work on B to contend and trigger wound flow. + burn(hold_iters); + log_entry(ctx, "sent_to_b", &payload); Ok(()) } @@ -90,6 +95,7 @@ pub fn call_c_from_b( seq: u64, message: String, burn_iters: u64, + hold_iters: u64, ) -> Result<(), String> { burn(burn_iters); @@ -104,6 +110,10 @@ pub fn call_c_from_b( spacetimedb::remote_reducer::call_reducer_on_db_2pc(c, "record_on_c", &args) .map_err(|e| format!("call_c_from_b: call to C failed: {e}"))?; + // Hold B open after C is prepared so B remains the global-tx owner while + // A-originated work attempts to prepare on B. + burn(hold_iters); + log_entry(ctx, "sent_to_c", &payload); Ok(()) } From 04d7fc664b5e4783131cc2e576ccacde220123f9 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Tue, 31 Mar 2026 15:31:24 -0700 Subject: [PATCH 28/33] Some fixes with a hack for retries --- crates/core/src/host/global_tx.rs | 154 ++++++++++++++++++++-------- modules/chain-call-repro/run.sh | 29 +++++- modules/chain-call-repro/src/lib.rs | 15 ++- 3 files changed, 150 insertions(+), 48 deletions(-) diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index 4036d03e27d..7fa275fd28d 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -174,7 +174,6 @@ impl<'a> WaitRegistration<'a> { fn disarm(mut self, ls: &mut std::sync::MutexGuard<'_, LockState>) { self.remove_waiter(ls); - self.wait_id = None; } fn remove_waiter(&mut self, ls: &mut LockState) { @@ -341,8 +340,16 @@ impl GlobalTxManager { if *wounded_rx.borrow() { return AcquireDisposition::Cancelled; } + if self.is_terminalish(&tx_id) { + return AcquireDisposition::Cancelled; + } - let (notify, owner_to_wound, new_registration): (Arc, Option, Option>) = { + let (notify, owner_to_wound, new_registration, cancelled): ( + Arc, + Option, + Option>, + bool, + ) = { let mut state = self.lock_state.lock().unwrap(); if state.owner.is_none() { self.prune_stale_head_waiters_locked(&mut state); @@ -359,17 +366,15 @@ impl GlobalTxManager { return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } None => { - let (wait_id, notify) = if let Some(registration) = registration.as_ref() { - let wait_id = registration.wait_id(); - let notify = state - .wait_entries - .get(&wait_id) - .expect("wait entry must exist for registered waiter") - .notify - .clone(); - (wait_id, notify) - } else { - self.ensure_waiter_locked(&mut state, tx_id) + let waiter = match registration.as_ref() { + Some(registration) => match self.registered_waiter_locked(&state, tx_id, registration) { + Ok(registered_waiter) => Some(registered_waiter), + Err(()) => None, + }, + None => Some(self.ensure_waiter_locked(&mut state, tx_id)), + }; + let Some((wait_id, notify)) = waiter else { + return AcquireDisposition::Cancelled; }; let head_waiter = state.waiting.first().map(|wait_key| wait_key.tx_id); if let Some(head_waiter) = head_waiter @@ -386,7 +391,7 @@ impl GlobalTxManager { head_waiter ); let new_registration = registration.is_none().then(|| WaitRegistration::new(self, wait_id)); - (notify, None, new_registration) + (notify, None, new_registration, false) } Some(owner) if owner == tx_id => { log::warn!("global transaction {tx_id} is trying to acquire the lock it already holds. This should not happen and may indicate a bug in the caller's logic, but we'll allow it to proceed without deadlocking on itself."); @@ -397,24 +402,25 @@ impl GlobalTxManager { return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); } Some(owner) => { - let (wait_id, notify) = if let Some(registration) = registration.as_ref() { - let wait_id = registration.wait_id(); - let notify = state - .wait_entries - .get(&wait_id) - .expect("wait entry must exist for registered waiter") - .notify - .clone(); - (wait_id, notify) - } else { - self.ensure_waiter_locked(&mut state, tx_id) + let waiter = match registration.as_ref() { + Some(registration) => match self.registered_waiter_locked(&state, tx_id, registration) { + Ok(registered_waiter) => Some(registered_waiter), + Err(()) => None, + }, + None => Some(self.ensure_waiter_locked(&mut state, tx_id)), + }; + let Some((wait_id, notify)) = waiter else { + return AcquireDisposition::Cancelled; }; let owner_to_wound = (tx_id < owner && state.wounded_owners.insert(owner)).then_some(owner); let new_registration = registration.is_none().then(|| WaitRegistration::new(self, wait_id)); - (notify, owner_to_wound, new_registration) + (notify, owner_to_wound, new_registration, false) } } }; + if cancelled { + return AcquireDisposition::Cancelled; + } if let Some(new_registration) = new_registration { registration = Some(new_registration); } @@ -514,6 +520,23 @@ impl GlobalTxManager { (wait_id, notify) } + fn registered_waiter_locked( + &self, + state: &LockState, + tx_id: GlobalTxId, + registration: &WaitRegistration<'_>, + ) -> Result<(u64, Arc), ()> { + let wait_id = registration.wait_id(); + if let Some(wait_entry) = state.wait_entries.get(&wait_id) { + return Ok((wait_id, wait_entry.notify.clone())); + } + + log::warn!( + "global transaction {tx_id} lost its waiter registration while still waiting; treating acquire as cancelled" + ); + Err(()) + } + fn is_next_waiter_locked(&self, state: &LockState, tx_id: GlobalTxId) -> bool { match state.waiting.first() { None => true, @@ -561,24 +584,23 @@ impl GlobalTxManager { .unwrap_or(true) } + fn is_terminalish(&self, tx_id: &GlobalTxId) -> bool { + let Some(session) = self.get_session(tx_id) else { + return true; + }; + session.is_wounded() + || matches!( + session.state(), + GlobalTxState::Committed | GlobalTxState::Aborted | GlobalTxState::Aborting + ) + } + fn prune_stale_head_waiters_locked(&self, state: &mut LockState) { while let Some(wait_key) = state.waiting.first().copied() { let tx_id = wait_key.tx_id; - let Some(session) = self.get_session(&tx_id) else { - log::warn!( - "pruning stale head waiter {tx_id}: no session exists while owner is None" - ); - self.remove_waiter_by_id(state, wait_key.wait_id); - continue; - }; - let session_state = session.state(); - let wounded = session.is_wounded(); - let terminalish = wounded - || matches!( - session_state, - GlobalTxState::Committed | GlobalTxState::Aborted | GlobalTxState::Aborting - ); - if terminalish { + if self.is_terminalish(&tx_id) { + let session_state = self.get_session(&tx_id).map(|session| session.state()); + let wounded = self.is_wounded(&tx_id); log::warn!( "pruning stale head waiter {tx_id}: state={session_state:?} wounded={wounded} while owner is None" ); @@ -830,4 +852,54 @@ mod tests { assert!(matches!(rt.block_on(waiter_task).expect("task should complete"), true)); } + + #[test] + fn pruned_waiter_is_cancelled_instead_of_panicking() { + let manager = Arc::new(GlobalTxManager::default()); + let owner = tx_id(20, 2, 0); + let waiter = tx_id(10, 1, 0); + manager.ensure_session(owner, super::GlobalTxRole::Participant, owner.creator_db); + manager.ensure_session(waiter, super::GlobalTxRole::Participant, waiter.creator_db); + + let rt = Runtime::new().unwrap(); + let owner_guard = match rt.block_on(manager.acquire(owner, |_| async {})) { + AcquireDisposition::Acquired(guard) => guard, + AcquireDisposition::Cancelled => panic!("owner should acquire immediately"), + }; + + let manager_for_task = manager.clone(); + let waiter_task = rt.spawn(async move { + matches!( + manager_for_task.acquire(waiter, |_| async {}).await, + AcquireDisposition::Cancelled + ) + }); + + let deadline = std::time::Instant::now() + Duration::from_millis(100); + while std::time::Instant::now() < deadline { + if manager + .lock_state + .lock() + .unwrap() + .waiter_ids_by_tx + .contains_key(&waiter) + { + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + assert!( + manager + .lock_state + .lock() + .unwrap() + .waiter_ids_by_tx + .contains_key(&waiter), + "waiter should be registered before pruning it", + ); + manager.mark_state(&waiter, super::GlobalTxState::Aborting); + drop(owner_guard); + + assert!(rt.block_on(waiter_task).expect("task should complete")); + } } diff --git a/modules/chain-call-repro/run.sh b/modules/chain-call-repro/run.sh index 80753472e13..7d1fa8074ae 100755 --- a/modules/chain-call-repro/run.sh +++ b/modules/chain-call-repro/run.sh @@ -116,9 +116,14 @@ run_a_client() { local client_id="$1" local failures=0 local log_file + local failure_log_file + local tmp_output local seq log_file="$TMP_DIR/a-client-${client_id}.log" + failure_log_file="$TMP_DIR/a-client-${client_id}.failures.log" + tmp_output="$TMP_DIR/a-client-${client_id}.tmp" + : >"$failure_log_file" seq=1 while :; do if ! ( @@ -130,9 +135,15 @@ run_a_client() { "a-msg-client-${client_id}-seq-${seq}" \ "$BURN_ITERS" \ "$HOLD_ITERS" - ) >"$log_file" 2>&1; then + ) >"$tmp_output" 2>&1; then failures=$((failures + 1)) + { + echo "=== failure $failures for a-client-${client_id} seq $seq ===" + cat "$tmp_output" + echo + } >>"$failure_log_file" fi + mv "$tmp_output" "$log_file" if [[ "$RUN_FOREVER" -eq 0 && "$seq" -ge "$ITERATIONS" ]]; then break @@ -147,9 +158,14 @@ run_b_client() { local client_id="$1" local failures=0 local log_file + local failure_log_file + local tmp_output local seq log_file="$TMP_DIR/b-client-${client_id}.log" + failure_log_file="$TMP_DIR/b-client-${client_id}.failures.log" + tmp_output="$TMP_DIR/b-client-${client_id}.tmp" + : >"$failure_log_file" seq=1 while :; do if ! ( @@ -161,9 +177,15 @@ run_b_client() { "b-msg-client-${client_id}-seq-${seq}" \ "$BURN_ITERS" \ "$HOLD_ITERS" - ) >"$log_file" 2>&1; then + ) >"$tmp_output" 2>&1; then failures=$((failures + 1)) + { + echo "=== failure $failures for b-client-${client_id} seq $seq ===" + cat "$tmp_output" + echo + } >>"$failure_log_file" fi + mv "$tmp_output" "$log_file" if [[ "$RUN_FOREVER" -eq 0 && "$seq" -ge "$ITERATIONS" ]]; then break @@ -260,8 +282,7 @@ fi if [[ "$TOTAL_FAILURES" -ne 0 ]]; then echo echo "At least one client call failed. Sample failure logs:" - find "$TMP_DIR" -name '*-client-*.log' -type f -print0 \ - | xargs -0 grep -l "Error\|failed\|panic" \ + find "$TMP_DIR" -name '*-client-*.failures.log' -type f -size +0c -print \ | head -n 10 \ | while read -r log_file; do echo "--- $log_file ---" diff --git a/modules/chain-call-repro/src/lib.rs b/modules/chain-call-repro/src/lib.rs index 8ff8047fd14..bd2f1631779 100644 --- a/modules/chain-call-repro/src/lib.rs +++ b/modules/chain-call-repro/src/lib.rs @@ -1,4 +1,7 @@ -use spacetimedb::{Identity, ReducerContext, SpacetimeType, Table}; +use spacetimedb::{ + remote_reducer::{into_reducer_error_message, RemoteCallError}, + Identity, ReducerContext, SpacetimeType, Table, +}; #[derive(SpacetimeType, Clone)] pub struct CallPayload { @@ -77,7 +80,10 @@ pub fn call_b_from_a( let args = spacetimedb::spacetimedb_lib::bsatn::to_vec(&(payload.clone(), burn_iters)) .expect("failed to encode args for record_on_b"); spacetimedb::remote_reducer::call_reducer_on_db_2pc(b, "record_on_b", &args) - .map_err(|e| format!("call_b_from_a: call to B failed: {e}"))?; + .map_err(|e| match e { + RemoteCallError::Wounded(_) => into_reducer_error_message(e), + _ => format!("call_b_from_a: call to B failed: {e}"), + })?; // Hold A open after B is prepared so B keeps its global-tx admission lock // long enough for concurrent work on B to contend and trigger wound flow. @@ -108,7 +114,10 @@ pub fn call_c_from_b( let args = spacetimedb::spacetimedb_lib::bsatn::to_vec(&(payload.clone(), burn_iters)) .expect("failed to encode args for record_on_c"); spacetimedb::remote_reducer::call_reducer_on_db_2pc(c, "record_on_c", &args) - .map_err(|e| format!("call_c_from_b: call to C failed: {e}"))?; + .map_err(|e| match e { + RemoteCallError::Wounded(_) => into_reducer_error_message(e), + _ => format!("call_c_from_b: call to C failed: {e}"), + })?; // Hold B open after C is prepared so B remains the global-tx owner while // A-originated work attempts to prepare on B. From f1f8740ff67a81d1a66ac8ab31c7b274b2e66e2b Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Tue, 31 Mar 2026 16:21:08 -0700 Subject: [PATCH 29/33] Clean up lock guard --- crates/core/src/host/global_tx.rs | 31 ++++----- crates/core/src/host/module_host.rs | 67 +++++++------------ .../src/host/wasm_common/module_host_actor.rs | 5 +- 3 files changed, 40 insertions(+), 63 deletions(-) diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index 7fa275fd28d..a9e31510361 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -145,13 +145,13 @@ struct WaitEntry { notify: Arc, } -pub enum AcquireDisposition<'a> { - Acquired(GlobalTxLockGuard<'a>), +pub enum AcquireDisposition { + Acquired(GlobalTxLockGuard), Cancelled, } -pub struct GlobalTxLockGuard<'a> { - manager: &'a GlobalTxManager, +pub struct GlobalTxLockGuard { + manager: Arc, tx_id: Option, } @@ -193,8 +193,8 @@ impl Drop for WaitRegistration<'_> { } } -impl<'a> GlobalTxLockGuard<'a> { - fn new(manager: &'a GlobalTxManager, tx_id: GlobalTxId) -> Self { +impl GlobalTxLockGuard { + fn new(manager: Arc, tx_id: GlobalTxId) -> Self { Self { manager, tx_id: Some(tx_id), @@ -204,14 +204,9 @@ impl<'a> GlobalTxLockGuard<'a> { pub fn tx_id(&self) -> GlobalTxId { self.tx_id.expect("lock guard must always have a tx_id before drop") } - - pub fn disarm(mut self) { - log::warn!("Disarming a lock guard without releasing the lock for tx_id {}", self.tx_id()); - self.tx_id = None; - } } -impl Drop for GlobalTxLockGuard<'_> { +impl Drop for GlobalTxLockGuard { fn drop(&mut self) { if let Some(tx_id) = self.tx_id.take() { self.manager.release(&tx_id); @@ -326,7 +321,7 @@ impl GlobalTxManager { Some(session) } - pub async fn acquire(&self, tx_id: GlobalTxId, mut on_wound: F) -> AcquireDisposition<'_> + pub async fn acquire(self: &Arc, tx_id: GlobalTxId, mut on_wound: F) -> AcquireDisposition where F: FnMut(GlobalTxId) -> Fut, Fut: Future + Send + 'static, @@ -363,7 +358,7 @@ impl GlobalTxManager { registration.disarm(&mut state); } log::info!("global transaction {tx_id} acquired the lock"); - return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); + return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self.clone(), tx_id)); } None => { let waiter = match registration.as_ref() { @@ -399,7 +394,7 @@ impl GlobalTxManager { if let Some(registration) = registration.take() { registration.disarm(&mut state); } - return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self, tx_id)); + return AcquireDisposition::Acquired(GlobalTxLockGuard::new(self.clone(), tx_id)); } Some(owner) => { let waiter = match registration.as_ref() { @@ -695,7 +690,7 @@ mod tests { #[test] fn younger_requester_waits_behind_older_owner() { - let manager = GlobalTxManager::default(); + let manager = Arc::new(GlobalTxManager::default()); let older = tx_id(10, 1, 0); let younger = tx_id(20, 2, 0); manager.ensure_session(older, super::GlobalTxRole::Participant, older.creator_db); @@ -743,7 +738,7 @@ mod tests { #[test] fn wound_is_idempotent() { - let manager = GlobalTxManager::default(); + let manager = Arc::new(GlobalTxManager::default()); let tx_id = tx_id(10, 1, 0); let session = manager.ensure_session(tx_id, super::GlobalTxRole::Coordinator, tx_id.creator_db); @@ -756,7 +751,7 @@ mod tests { #[test] fn wound_subscription_notifies_waiter() { - let manager = GlobalTxManager::default(); + let manager = Arc::new(GlobalTxManager::default()); let tx_id = tx_id(10, 1, 0); let _session = manager.ensure_session(tx_id, super::GlobalTxRole::Coordinator, tx_id.creator_db); let mut wounded_rx = manager.subscribe_wounded(&tx_id).expect("session should exist"); diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 6b8ffa49f12..d930947981c 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -233,24 +233,6 @@ pub struct ModuleInfo { pub metrics: ModuleMetrics, } -struct GlobalTxAdmissionGuard<'a> { - lock_guard: Option>, -} - -impl<'a> GlobalTxAdmissionGuard<'a> { - fn new(lock_guard: super::global_tx::GlobalTxLockGuard<'a>) -> Self { - Self { - lock_guard: Some(lock_guard), - } - } - - fn disarm(mut self) { - if let Some(lock_guard) = self.lock_guard.take() { - lock_guard.disarm(); - } - } -} - impl fmt::Debug for ModuleInfo { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ModuleInfo") @@ -1640,7 +1622,7 @@ impl ModuleHost { let args = args .into_tuple_for_def(&self.info.module_def, reducer_def) .map_err(InvalidReducerArguments)?; - let admission_guard = if let Some(tx_id) = tx_id { + let _global_tx_lock_guard = if let Some(tx_id) = tx_id { match self.acquire_global_tx_slot(tx_id).await { Ok(guard) => Some(guard), Err(outcome) => { @@ -1675,9 +1657,6 @@ impl ModuleHost { async |p, inst| inst.call_reducer(p).await, ) .await; - if let Some(guard) = admission_guard { - guard.disarm(); - } result? } @@ -1696,7 +1675,7 @@ impl ModuleHost { let args = args .into_tuple_for_def(&self.info.module_def, reducer_def) .map_err(InvalidReducerArguments)?; - let admission_guard = if let Some(tx_id) = tx_id { + let _global_tx_lock_guard = if let Some(tx_id) = tx_id { match self.acquire_global_tx_slot(tx_id).await { Ok(guard) => Some(guard), Err(outcome) => { @@ -1734,9 +1713,6 @@ impl ModuleHost { async |p, inst| inst.call_reducer(p).await.map(|res| (res, None)), ) .await; - if let Some(guard) = admission_guard { - guard.disarm(); - } result? } @@ -1930,10 +1906,8 @@ impl ModuleHost { self.replica_ctx() .global_tx_manager .set_prepare_mapping(tx_id, prepare_id.clone()); - match self.acquire_global_tx_slot(tx_id).await { - Ok(guard) => { - guard.disarm(); - } + let global_tx_lock_guard = match self.acquire_global_tx_slot(tx_id).await { + Ok(guard) => guard, Err(outcome) => { self.prepared_txs.remove(&prepare_id); self.replica_ctx().global_tx_manager.remove_prepare_mapping(&prepare_id); @@ -1952,7 +1926,7 @@ impl ModuleHost { None, )); } - } + }; //} // Spawn a background task that runs the reducer and holds the write lock @@ -1965,13 +1939,20 @@ impl ModuleHost { let _ = this .call( &reducer_name_owned, - (params, prepare_id_clone, coordinator_identity, prepared_tx, decision_rx), - async |(p, pid, cid, ptx, drx), inst| { - inst.call_reducer_prepare_and_hold(p, pid, cid, ptx, drx); + ( + params, + prepare_id_clone, + coordinator_identity, + prepared_tx, + decision_rx, + global_tx_lock_guard, + ), + async |(p, pid, cid, ptx, drx, guard), inst| { + inst.call_reducer_prepare_and_hold(p, pid, cid, ptx, drx, guard); Ok::<(), ReducerCallError>(()) }, // JS modules: no 2PC support yet. - async |(_p, _pid, _cid, _ptx, _drx), _inst| Err(ReducerCallError::NoSuchReducer), + async |(_p, _pid, _cid, _ptx, _drx, _guard), _inst| Err(ReducerCallError::NoSuchReducer), ) .await; }); @@ -1994,7 +1975,6 @@ impl ModuleHost { self.replica_ctx() .global_tx_manager .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborted); - self.replica_ctx().global_tx_manager.release(&tx_id); self.replica_ctx().global_tx_manager.remove_session(&tx_id); // } Ok((String::new(), result, return_value)) @@ -2120,8 +2100,8 @@ impl ModuleHost { Ok(()) } - async fn acquire_global_tx_slot(&self, tx_id: GlobalTxId) -> Result, ReducerOutcome> { - let manager = &self.replica_ctx().global_tx_manager; + async fn acquire_global_tx_slot(&self, tx_id: GlobalTxId) -> Result { + let manager = self.replica_ctx().global_tx_manager.clone(); let local_db = self.replica_ctx().database.database_identity; let role = if tx_id.creator_db == local_db { super::global_tx::GlobalTxRole::Coordinator @@ -2150,10 +2130,11 @@ impl ModuleHost { super::global_tx::AcquireDisposition::Acquired(lock_guard) => { if let Some(outcome) = self.check_global_tx_wounded(tx_id) { log::info!("global transaction {tx_id} was wounded immediately after scheduler admission"); - self.abort_global_tx_locally(tx_id, true); + drop(lock_guard); + self.finish_abort_global_tx_locally(tx_id, true); return Err(outcome); } - return Ok(GlobalTxAdmissionGuard::new(lock_guard)); + return Ok(lock_guard); } super::global_tx::AcquireDisposition::Cancelled => { log::info!("global transaction {tx_id} was cancelled while waiting for scheduler admission"); @@ -2177,10 +2158,14 @@ impl ModuleHost { "global transaction {tx_id} aborting locally on {}; remove_session={remove_session}", self.replica_ctx().database.database_identity ); + self.replica_ctx().global_tx_manager.release(&tx_id); + self.finish_abort_global_tx_locally(tx_id, remove_session); + } + + fn finish_abort_global_tx_locally(&self, tx_id: GlobalTxId, remove_session: bool) { self.replica_ctx() .global_tx_manager .mark_state(&tx_id, super::global_tx::GlobalTxState::Aborted); - self.replica_ctx().global_tx_manager.release(&tx_id); if remove_session { self.replica_ctx().global_tx_manager.remove_session(&tx_id); } diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 2af85569dcc..fde84281799 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -653,6 +653,7 @@ impl WasmModuleInstance { coordinator_identity: crate::identity::Identity, prepared_tx: tokio::sync::oneshot::Sender<(ReducerCallResult, Option)>, decision_rx: std::sync::mpsc::Receiver, + _global_tx_lock_guard: crate::host::global_tx::GlobalTxLockGuard, ) { let stdb = self.instance.replica_ctx().relational_db().clone(); let replica_ctx = self.instance.replica_ctx().clone(); @@ -701,7 +702,6 @@ impl WasmModuleInstance { replica_ctx .global_tx_manager .mark_state(&tx_id, crate::host::global_tx::GlobalTxState::Aborted); - replica_ctx.global_tx_manager.release(&tx_id); replica_ctx.global_tx_manager.remove_session(&tx_id); } return; @@ -789,7 +789,6 @@ impl WasmModuleInstance { replica_ctx .global_tx_manager .mark_state(&tx_id, crate::host::global_tx::GlobalTxState::Committed); - replica_ctx.global_tx_manager.release(&tx_id); replica_ctx.global_tx_manager.remove_session(&tx_id); } } else { @@ -812,7 +811,6 @@ impl WasmModuleInstance { replica_ctx .global_tx_manager .mark_state(&tx_id, crate::host::global_tx::GlobalTxState::Aborted); - replica_ctx.global_tx_manager.release(&tx_id); replica_ctx.global_tx_manager.remove_session(&tx_id); } } @@ -1333,7 +1331,6 @@ impl InstanceCommon { crate::host::global_tx::GlobalTxState::Aborted }, ); - manager.release(&tx_id); manager.remove_session(&tx_id); } From 3ab5eb0958073c73613f731cc2cee31499925493 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Wed, 1 Apr 2026 08:44:32 -0700 Subject: [PATCH 30/33] More metrics --- crates/client-api/src/routes/database.rs | 15 ++++++- crates/core/src/host/global_tx.rs | 30 ++++++++++++++ crates/core/src/host/instance_env.rs | 4 ++ crates/core/src/host/module_host.rs | 5 +++ .../src/host/wasm_common/module_host_actor.rs | 12 ++++++ crates/core/src/worker_metrics/mod.rs | 40 +++++++++++++++++++ 6 files changed, 105 insertions(+), 1 deletion(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 746652dad82..2f1d0d16bff 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -28,6 +28,7 @@ use serde::Deserialize; use spacetimedb::database_logger::DatabaseLogger; use spacetimedb::host::module_host::ClientConnectedError; use spacetimedb::host::{CallResult, UpdateDatabaseResult}; +use spacetimedb::worker_metrics::WORKER_METRICS; use spacetimedb::host::{FunctionArgs, MigratePlanResult}; use spacetimedb::host::{ModuleHost, ReducerOutcome}; use spacetimedb::host::{ProcedureCallError, ReducerCallError}; @@ -327,7 +328,19 @@ pub async fn prepare( .into()); } - let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; + let ( + module, + Database { + owner_identity, + database_identity, + .. + }, + ) = find_module_and_database(&worker_ctx, name_or_identity).await?; + + WORKER_METRICS + .two_pc_prepare_calls_received_total + .with_label_values(&database_identity) + .inc(); // 2PC prepare is a server-to-server call; no client lifecycle management needed. // call_identity_connected/disconnected submit jobs to the module's executor, which diff --git a/crates/core/src/host/global_tx.rs b/crates/core/src/host/global_tx.rs index a9e31510361..3605d8c45fe 100644 --- a/crates/core/src/host/global_tx.rs +++ b/crates/core/src/host/global_tx.rs @@ -1,4 +1,5 @@ use crate::identity::Identity; +use crate::worker_metrics::WORKER_METRICS; use spacetimedb_lib::GlobalTxId; use std::cmp::Ordering as CmpOrdering; use std::collections::{BTreeSet, HashMap, HashSet}; @@ -228,6 +229,15 @@ impl Default for GlobalTxManager { } impl GlobalTxManager { + fn session_metric_labels(&self, tx_id: &GlobalTxId) -> Option<(Identity, &'static str)> { + let session = self.get_session(tx_id)?; + let role = match session.role { + GlobalTxRole::Coordinator => "coordinator", + GlobalTxRole::Participant => "participant", + }; + Some((session.coordinator_identity, role)) + } + pub fn new(wound_grace_period: Duration) -> Self { Self { sessions: Mutex::default(), @@ -312,11 +322,19 @@ impl GlobalTxManager { session.set_state(GlobalTxState::Aborting); } if was_fresh { + let role = match session.role { + GlobalTxRole::Coordinator => "coordinator", + GlobalTxRole::Participant => "participant", + }; log::info!( "global transaction {tx_id} marked wounded; role={:?} coordinator={}", session.role, session.coordinator_identity ); + WORKER_METRICS + .transactions_wounded_total + .with_label_values(&session.coordinator_identity, &role) + .inc(); } Some(session) } @@ -421,6 +439,12 @@ impl GlobalTxManager { } if let Some(owner) = owner_to_wound { + if let Some((coordinator_identity, role)) = self.session_metric_labels(&tx_id) { + WORKER_METRICS + .global_tx_waiting_on_younger_owner_total + .with_label_values(&coordinator_identity, &role) + .inc(); + } let wound_grace_period = self.wound_grace_period; log::info!( "global transaction {tx_id} is waiting behind younger owner {owner}; giving it {:?} to finish before wound flow", @@ -437,6 +461,12 @@ impl GlobalTxManager { _ = tokio::time::sleep(wound_grace_period) => false, }; if owner_finished { + if let Some((coordinator_identity, role)) = self.session_metric_labels(&tx_id) { + WORKER_METRICS + .global_tx_younger_owner_finished_within_grace_period_total + .with_label_values(&coordinator_identity, &role) + .inc(); + } log::info!("global transaction {tx_id} observed owner {owner} finish within grace period; not triggering wound",); continue; } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 1ec455d1371..6be7b9f1a91 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1182,6 +1182,10 @@ impl InstanceEnv { .cross_db_reducer_calls_total .with_label_values(&caller_identity) .inc(); + WORKER_METRICS + .two_pc_outgoing_prepare_calls_total + .with_label_values(&caller_identity) + .inc(); WORKER_METRICS .cross_db_reducer_duration_seconds .with_label_values(&caller_identity) diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index d930947981c..3f75a4f6c1f 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2192,6 +2192,7 @@ impl ModuleHost { let client = self.replica_ctx().call_reducer_client.clone(); let router = self.replica_ctx().call_reducer_router.clone(); let auth_token = self.replica_ctx().call_reducer_auth_token.clone(); + let local_db = self.replica_ctx().database.database_identity; let base_url = match router.resolve_base_url(tx_id.creator_db).await { Ok(url) => url, Err(e) => { @@ -2210,6 +2211,10 @@ impl ModuleHost { req = req.header(http::header::AUTHORIZATION, format!("Bearer {token}")); } log::info!("2PC wound: sending wound for {tx_id} to coordinator at {url}"); + WORKER_METRICS + .two_pc_wound_requests_sent_total + .with_label_values(&local_db) + .inc(); match req.send().await { Ok(resp) if resp.status().is_success() => { log::info!("2PC wound: notified coordinator for {tx_id}"); diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index fde84281799..796d0c11915 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -760,6 +760,10 @@ impl WasmModuleInstance { ); if commit { + WORKER_METRICS + .two_pc_transactions_committed_as_participant_total + .with_label_values(&replica_ctx.database.database_identity) + .inc(); if let Some(tx_id) = global_tx_id { log::info!( "2PC participant {} committing prepared transaction {tx_id} ({prepare_id})", @@ -1238,6 +1242,14 @@ impl InstanceCommon { let committed = matches!(event.status, EventStatus::Committed(_)); let stdb = self.info.subscriptions.relational_db().clone(); let handle = tokio::runtime::Handle::current(); + let local_db = inst.replica_ctx().database.database_identity; + + if committed { + WORKER_METRICS + .two_pc_transactions_committed_total + .with_label_values(&local_db) + .inc(); + } // Wait for A's coordinator log (committed atomically with the tx) to be // durable before sending COMMIT to B. This guarantees that if A crashes diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index 5b84e230045..062e032e5b0 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -48,11 +48,51 @@ metrics_group!( #[labels(caller_identity: Identity)] pub cross_db_reducer_calls_total: IntCounterVec, + #[name = spacetime_2pc_outgoing_prepare_calls_total] + #[help = "Total number of outgoing 2PC prepare calls made by this database while acting as coordinator."] + #[labels(database_identity: Identity)] + pub two_pc_outgoing_prepare_calls_total: IntCounterVec, + + #[name = spacetime_2pc_prepare_calls_received_total] + #[help = "Total number of incoming 2PC prepare calls received by this database."] + #[labels(database_identity: Identity)] + pub two_pc_prepare_calls_received_total: IntCounterVec, + + #[name = spacetime_2pc_transactions_committed_total] + #[help = "Total number of 2PC transactions committed by this database as coordinator after preparing remote participants."] + #[labels(database_identity: Identity)] + pub two_pc_transactions_committed_total: IntCounterVec, + + #[name = spacetime_2pc_transactions_committed_as_participant_total] + #[help = "Total number of 2PC transactions committed by this database as a participant."] + #[labels(database_identity: Identity)] + pub two_pc_transactions_committed_as_participant_total: IntCounterVec, + #[name = spacetime_cross_db_reducer_duration_seconds] #[help = "Duration of cross-database reducer calls in seconds."] #[labels(caller_identity: Identity)] pub cross_db_reducer_duration_seconds: HistogramVec, + #[name = spacetime_transactions_wounded_total] + #[help = "Total number of distributed transactions that were marked wounded."] + #[labels(coordinator_identity: Identity, role: str)] + pub transactions_wounded_total: IntCounterVec, + + #[name = spacetime_2pc_wound_requests_sent_total] + #[help = "Total number of wound requests sent to a remote coordinator."] + #[labels(database_identity: Identity)] + pub two_pc_wound_requests_sent_total: IntCounterVec, + + #[name = spacetime_global_tx_waiting_on_younger_owner_total] + #[help = "Total number of times a transaction tried to acquire the global transaction lock and found itself behind a younger owner."] + #[labels(coordinator_identity: Identity, role: str)] + pub global_tx_waiting_on_younger_owner_total: IntCounterVec, + + #[name = spacetime_global_tx_younger_owner_finished_within_grace_period_total] + #[help = "Total number of times a younger lock owner finished within the wound grace period after blocking an older transaction."] + #[labels(coordinator_identity: Identity, role: str)] + pub global_tx_younger_owner_finished_within_grace_period_total: IntCounterVec, + #[name = jemalloc_active_bytes] #[help = "Number of bytes in jemallocs heap"] #[labels(node_id: str)] From 1886e7d584fd1e2ea0cbc2f0563ce4d2a5d46201 Mon Sep 17 00:00:00 2001 From: Jeffrey Dallatezza Date: Wed, 1 Apr 2026 09:09:08 -0700 Subject: [PATCH 31/33] Take tools/, modules/tpcc, modules/tpcc-results, and modules/tpcc-metrics from tyler/cycle-detection at 3c4f15b4b6f4c3256c8c7f4854f88f10112f0744 --- modules/tpcc-metrics/src/lib.rs | 91 +++- .../tpcc-1774908053099/summary.json | 348 ++++++++++++++++ .../tpcc-1774908671807/summary.json | 348 ++++++++++++++++ .../tpcc-1774909349292/summary.json | 348 ++++++++++++++++ .../tpcc-1774910938757/summary.json | 348 ++++++++++++++++ .../tpcc-1774911348130/summary.json | 348 ++++++++++++++++ .../driver-110838/txn_events.ndjson | 0 .../driver-135613/summary.json | 354 ++++++++++++++++ .../driver-135613/txn_events.ndjson | 129 ++++++ .../driver-137679/summary.json | 354 ++++++++++++++++ .../driver-137679/txn_events.ndjson | 151 +++++++ .../driver-138825/txn_events.ndjson | 30 ++ .../driver-152145/summary.json | 354 ++++++++++++++++ .../driver-152145/txn_events.ndjson | 143 +++++++ .../driver-179375/summary.json | 354 ++++++++++++++++ .../driver-179375/txn_events.ndjson | 137 ++++++ .../driver-184437/summary.json | 354 ++++++++++++++++ .../driver-184437/txn_events.ndjson | 144 +++++++ modules/tpcc/src/lib.rs | 9 +- modules/tpcc/src/load.rs | 75 +++- modules/tpcc/src/new_order.rs | 34 +- modules/tpcc/src/payment.rs | 58 ++- modules/tpcc/src/remote.rs | 55 ++- tools/tpcc-dashboard/.editorconfig | 4 + tools/tpcc-dashboard/index.html | 23 ++ tools/tpcc-dashboard/package-lock.json | 2 + tools/tpcc-dashboard/package.json | 2 + .../ibm-plex-mono-latin-600-normal.woff2 | Bin 0 -> 15620 bytes .../public/inter-latin-wght-normal.woff2 | Bin 0 -> 48256 bytes .../source-code-pro-latin-wght-normal.woff2 | Bin 0 -> 22044 bytes tools/tpcc-dashboard/src/App.css | 5 + tools/tpcc-dashboard/src/App.tsx | 115 +++--- tools/tpcc-dashboard/src/ConnectedGuard.tsx | 6 +- tools/tpcc-dashboard/src/Icons.tsx | 143 +++++++ .../src/LatencyDistributionChart.tsx | 53 +++ .../src/NewOrderThroughputChart.css | 22 + .../src/NewOrderThroughtputChart.tsx | 163 ++++---- tools/tpcc-dashboard/src/StatsCards.css | 23 ++ tools/tpcc-dashboard/src/StatsCards.tsx | 124 ++++++ tools/tpcc-dashboard/src/context.ts | 4 +- .../src/features/globalState.ts | 97 +++++ tools/tpcc-dashboard/src/hooks.ts | 5 + tools/tpcc-dashboard/src/main.tsx | 12 +- .../src/module_bindings/index.ts | 32 +- ...educer.ts => record_txn_bucket_reducer.ts} | 0 .../src/module_bindings/record_txn_reducer.ts | 15 + .../src/module_bindings/reset_reducer.ts | 1 + .../src/module_bindings/state_table.ts | 3 +- .../src/module_bindings/txn_bucket_table.ts | 16 + .../src/module_bindings/txn_table.ts | 17 + .../src/module_bindings/types.ts | 16 +- .../src/module_bindings/types/reducers.ts | 6 +- tools/tpcc-dashboard/src/store.ts | 11 + tools/tpcc-dashboard/src/style.css | 71 ++++ tools/tpcc-runner/src/config.rs | 15 + tools/tpcc-runner/src/coordinator.rs | 101 +++-- tools/tpcc-runner/src/driver.rs | 389 +++++++----------- tools/tpcc-runner/src/loader.rs | 2 + .../clear_state_reducer.rs | 3 +- .../src/metrics_module_bindings/mod.rs | 62 ++- ...educer.rs => record_txn_bucket_reducer.rs} | 32 +- .../record_txn_reducer.rs | 68 +++ .../metrics_module_bindings/reset_reducer.rs | 23 +- .../src/metrics_module_bindings/state_type.rs | 9 +- .../txn_bucket_table.rs | 157 +++++++ .../txn_bucket_type.rs | 52 +++ .../src/metrics_module_bindings/txn_table.rs | 151 +++++++ .../src/metrics_module_bindings/txn_type.rs | 55 +++ .../tpcc_load_config_request_type.rs | 2 + 69 files changed, 6158 insertions(+), 520 deletions(-) create mode 100644 modules/tpcc-results/coordinator/tpcc-1774908053099/summary.json create mode 100644 modules/tpcc-results/coordinator/tpcc-1774908671807/summary.json create mode 100644 modules/tpcc-results/coordinator/tpcc-1774909349292/summary.json create mode 100644 modules/tpcc-results/coordinator/tpcc-1774910938757/summary.json create mode 100644 modules/tpcc-results/coordinator/tpcc-1774911348130/summary.json create mode 100644 modules/tpcc-results/tpcc-1774902157443/driver-110838/txn_events.ndjson create mode 100644 modules/tpcc-results/tpcc-1774908053099/driver-135613/summary.json create mode 100644 modules/tpcc-results/tpcc-1774908053099/driver-135613/txn_events.ndjson create mode 100644 modules/tpcc-results/tpcc-1774908671807/driver-137679/summary.json create mode 100644 modules/tpcc-results/tpcc-1774908671807/driver-137679/txn_events.ndjson create mode 100644 modules/tpcc-results/tpcc-1774909172568/driver-138825/txn_events.ndjson create mode 100644 modules/tpcc-results/tpcc-1774909349292/driver-152145/summary.json create mode 100644 modules/tpcc-results/tpcc-1774909349292/driver-152145/txn_events.ndjson create mode 100644 modules/tpcc-results/tpcc-1774910938757/driver-179375/summary.json create mode 100644 modules/tpcc-results/tpcc-1774910938757/driver-179375/txn_events.ndjson create mode 100644 modules/tpcc-results/tpcc-1774911348130/driver-184437/summary.json create mode 100644 modules/tpcc-results/tpcc-1774911348130/driver-184437/txn_events.ndjson create mode 100644 tools/tpcc-dashboard/public/ibm-plex-mono-latin-600-normal.woff2 create mode 100644 tools/tpcc-dashboard/public/inter-latin-wght-normal.woff2 create mode 100644 tools/tpcc-dashboard/public/source-code-pro-latin-wght-normal.woff2 create mode 100644 tools/tpcc-dashboard/src/App.css create mode 100644 tools/tpcc-dashboard/src/Icons.tsx create mode 100644 tools/tpcc-dashboard/src/LatencyDistributionChart.tsx create mode 100644 tools/tpcc-dashboard/src/NewOrderThroughputChart.css create mode 100644 tools/tpcc-dashboard/src/StatsCards.css create mode 100644 tools/tpcc-dashboard/src/StatsCards.tsx create mode 100644 tools/tpcc-dashboard/src/features/globalState.ts create mode 100644 tools/tpcc-dashboard/src/hooks.ts rename tools/tpcc-dashboard/src/module_bindings/{register_completed_order_reducer.ts => record_txn_bucket_reducer.ts} (100%) create mode 100644 tools/tpcc-dashboard/src/module_bindings/record_txn_reducer.ts create mode 100644 tools/tpcc-dashboard/src/module_bindings/txn_bucket_table.ts create mode 100644 tools/tpcc-dashboard/src/module_bindings/txn_table.ts create mode 100644 tools/tpcc-dashboard/src/store.ts create mode 100644 tools/tpcc-dashboard/src/style.css rename tools/tpcc-runner/src/metrics_module_bindings/{register_completed_order_reducer.rs => record_txn_bucket_reducer.rs} (58%) create mode 100644 tools/tpcc-runner/src/metrics_module_bindings/record_txn_reducer.rs create mode 100644 tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_table.rs create mode 100644 tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_type.rs create mode 100644 tools/tpcc-runner/src/metrics_module_bindings/txn_table.rs create mode 100644 tools/tpcc-runner/src/metrics_module_bindings/txn_type.rs diff --git a/modules/tpcc-metrics/src/lib.rs b/modules/tpcc-metrics/src/lib.rs index 96721512d34..823e2f0b13d 100644 --- a/modules/tpcc-metrics/src/lib.rs +++ b/modules/tpcc-metrics/src/lib.rs @@ -1,5 +1,7 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; +const BUCKET_SIZE_MS: u64 = 1_000; + #[table(accessor = state, public)] pub struct State { #[primary_key] @@ -9,46 +11,101 @@ pub struct State { pub run_end_ms: u64, pub measure_start_ms: u64, pub measure_end_ms: u64, + pub warehouse_count: u64, +} - pub order_count: u64, +#[table(accessor = txn, public)] +pub struct Txn { + #[primary_key] + #[auto_inc] + pub id: u64, pub measurement_time_ms: u64, + pub latency_ms: u16, } -#[reducer] -pub fn reset(ctx: &ReducerContext, warmup_duration_ms: u64, measure_start_ms: u64, measure_end_ms: u64) { +#[table(accessor = txn_bucket, public)] +pub struct TxnBucket { + #[primary_key] + pub bucket_start_ms: u64, + pub count: u64, +} + +fn clear_tables(ctx: &ReducerContext) { for row in ctx.db.state().iter() { - ctx.db.state().delete(row); + ctx.db.state().id().delete(row.id); + } + + for row in ctx.db.txn().iter() { + ctx.db.txn().id().delete(row.id); } + for row in ctx.db.txn_bucket().iter() { + ctx.db.txn_bucket() + .bucket_start_ms() + .delete(row.bucket_start_ms); + } +} + +#[reducer] +pub fn reset( + ctx: &ReducerContext, + warehouse_count: u64, + warmup_duration_ms: u64, + measure_start_ms: u64, + measure_end_ms: u64, +) { + clear_tables(ctx); + ctx.db.state().insert(State { id: 0, - order_count: 0, - measurement_time_ms: 0, run_start_ms: measure_start_ms - warmup_duration_ms, run_end_ms: measure_end_ms + warmup_duration_ms, measure_start_ms, measure_end_ms, + warehouse_count, }); } #[reducer] pub fn clear_state(ctx: &ReducerContext) { - for row in ctx.db.state().iter() { - ctx.db.state().delete(row); - } + clear_tables(ctx); } #[reducer] -pub fn register_completed_order(ctx: &ReducerContext) { - // We intentionally do not check if the current time is within the measurement window, - // this is the driver's reponsibility - +pub fn record_txn(ctx: &ReducerContext, latency_ms: u16) { let current_time_ms = ctx.timestamp.to_duration_since_unix_epoch().unwrap().as_millis() as u64; - let mut state = ctx.db.state().id().find(0).unwrap(); + ctx.db.txn().insert(Txn { + id: 0, + measurement_time_ms: current_time_ms, + latency_ms, + }); +} - state.order_count += 1; - state.measurement_time_ms = current_time_ms; +#[reducer] +pub fn record_txn_bucket(ctx: &ReducerContext) { + let current_time_ms = ctx + .timestamp + .to_duration_since_unix_epoch() + .unwrap() + .as_millis() as u64; + let Some(state) = ctx.db.state().id().find(0) else { + return; + }; - ctx.db.state().id().update(state); + let bucket_offset_ms = current_time_ms.saturating_sub(state.run_start_ms); + let bucket_start_ms = + state.run_start_ms + ((bucket_offset_ms / BUCKET_SIZE_MS) * BUCKET_SIZE_MS); + + if let Some(bucket) = ctx.db.txn_bucket().bucket_start_ms().find(bucket_start_ms) { + ctx.db.txn_bucket().bucket_start_ms().update(TxnBucket { + count: bucket.count.saturating_add(1), + ..bucket + }); + } else { + ctx.db.txn_bucket().insert(TxnBucket { + bucket_start_ms, + count: 1, + }); + } } diff --git a/modules/tpcc-results/coordinator/tpcc-1774908053099/summary.json b/modules/tpcc-results/coordinator/tpcc-1774908053099/summary.json new file mode 100644 index 00000000000..d942bebebd3 --- /dev/null +++ b/modules/tpcc-results/coordinator/tpcc-1774908053099/summary.json @@ -0,0 +1,348 @@ +{ + "run_id": "tpcc-1774908053099", + "driver_count": 1, + "drivers": [ + "driver-135613" + ], + "generated_at_ms": 1774908449186, + "total_transactions": 129, + "tpmc_like": 134.0, + "transaction_mix": { + "delivery": 3.10077519379845, + "new_order": 51.93798449612403, + "order_status": 5.426356589147287, + "payment": 37.2093023255814, + "stock_level": 2.3255813953488373 + }, + "conformance": { + "new_order_rollbacks": 0, + "new_order_total": 67, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 649, + "payment_remote": 0, + "payment_total": 48, + "payment_by_last_name": 31, + "order_status_by_last_name": 3, + "order_status_total": 7, + "delivery_queued": 4, + "delivery_completed": 5, + "delivery_processed_districts": 50, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 4, + "success": 4, + "failure": 0, + "mean_latency_ms": 8.75, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 9, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 4, + "sum_ms": 35, + "max_ms": 9 + } + }, + "new_order": { + "count": 67, + "success": 67, + "failure": 0, + "mean_latency_ms": 9.701492537313433, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 11, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 54, + 13, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 67, + "sum_ms": 650, + "max_ms": 11 + } + }, + "order_status": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 1.7142857142857142, + "p50_latency_ms": 2, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 2, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 12, + "max_ms": 2 + } + }, + "payment": { + "count": 48, + "success": 48, + "failure": 0, + "mean_latency_ms": 8.458333333333334, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 48, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 48, + "sum_ms": 406, + "max_ms": 10 + } + }, + "stock_level": { + "count": 3, + "success": 3, + "failure": 0, + "mean_latency_ms": 5.666666666666667, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 6, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 1, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 3, + "sum_ms": 17, + "max_ms": 6 + } + } + }, + "delivery": { + "queued": 4, + "completed": 5, + "pending": 0, + "processed_districts": 50, + "skipped_districts": 0, + "completion_mean_ms": 24.8, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 40, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 1, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 5, + "sum_ms": 124, + "max_ms": 40 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/coordinator/tpcc-1774908671807/summary.json b/modules/tpcc-results/coordinator/tpcc-1774908671807/summary.json new file mode 100644 index 00000000000..3b0bfc4d75e --- /dev/null +++ b/modules/tpcc-results/coordinator/tpcc-1774908671807/summary.json @@ -0,0 +1,348 @@ +{ + "run_id": "tpcc-1774908671807", + "driver_count": 1, + "drivers": [ + "driver-137679" + ], + "generated_at_ms": 1774909101311, + "total_transactions": 151, + "tpmc_like": 138.0, + "transaction_mix": { + "delivery": 3.9735099337748343, + "new_order": 45.6953642384106, + "order_status": 4.635761589403973, + "payment": 41.05960264900662, + "stock_level": 4.635761589403973 + }, + "conformance": { + "new_order_rollbacks": 0, + "new_order_total": 69, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 735, + "payment_remote": 0, + "payment_total": 62, + "payment_by_last_name": 39, + "order_status_by_last_name": 4, + "order_status_total": 7, + "delivery_queued": 6, + "delivery_completed": 10, + "delivery_processed_districts": 100, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 6, + "success": 6, + "failure": 0, + "mean_latency_ms": 8.166666666666666, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 6, + "sum_ms": 49, + "max_ms": 10 + } + }, + "new_order": { + "count": 69, + "success": 69, + "failure": 0, + "mean_latency_ms": 9.91304347826087, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 16, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 51, + 18, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 69, + "sum_ms": 684, + "max_ms": 16 + } + }, + "order_status": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 1.1428571428571428, + "p50_latency_ms": 1, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 6, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 8, + "max_ms": 2 + } + }, + "payment": { + "count": 62, + "success": 62, + "failure": 0, + "mean_latency_ms": 8.661290322580646, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 20, + "max_latency_ms": 11, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 60, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 62, + "sum_ms": 537, + "max_ms": 11 + } + }, + "stock_level": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 5.428571428571429, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 6, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 3, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 38, + "max_ms": 6 + } + } + }, + "delivery": { + "queued": 6, + "completed": 10, + "pending": 0, + "processed_districts": 100, + "skipped_districts": 0, + "completion_mean_ms": 21.0, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 23, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 4, + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 10, + "sum_ms": 210, + "max_ms": 23 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/coordinator/tpcc-1774909349292/summary.json b/modules/tpcc-results/coordinator/tpcc-1774909349292/summary.json new file mode 100644 index 00000000000..5b26641bdf2 --- /dev/null +++ b/modules/tpcc-results/coordinator/tpcc-1774909349292/summary.json @@ -0,0 +1,348 @@ +{ + "run_id": "tpcc-1774909349292", + "driver_count": 1, + "drivers": [ + "driver-152145" + ], + "generated_at_ms": 1774909758833, + "total_transactions": 143, + "tpmc_like": 132.0, + "transaction_mix": { + "delivery": 4.895104895104895, + "new_order": 48.25174825174825, + "order_status": 1.3986013986013985, + "payment": 42.65734265734266, + "stock_level": 2.797202797202797 + }, + "conformance": { + "new_order_rollbacks": 3, + "new_order_total": 69, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 719, + "payment_remote": 0, + "payment_total": 61, + "payment_by_last_name": 35, + "order_status_by_last_name": 0, + "order_status_total": 2, + "delivery_queued": 7, + "delivery_completed": 9, + "delivery_processed_districts": 90, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 7.857142857142857, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 55, + "max_ms": 10 + } + }, + "new_order": { + "count": 69, + "success": 66, + "failure": 3, + "mean_latency_ms": 9.782608695652174, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 19, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 1, + 2, + 0, + 43, + 23, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 69, + "sum_ms": 675, + "max_ms": 19 + } + }, + "order_status": { + "count": 2, + "success": 2, + "failure": 0, + "mean_latency_ms": 1.0, + "p50_latency_ms": 1, + "p95_latency_ms": 1, + "p99_latency_ms": 1, + "max_latency_ms": 1, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 2, + "sum_ms": 2, + "max_ms": 1 + } + }, + "payment": { + "count": 61, + "success": 61, + "failure": 0, + "mean_latency_ms": 8.721311475409836, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 20, + "max_latency_ms": 14, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 59, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 61, + "sum_ms": 532, + "max_ms": 14 + } + }, + "stock_level": { + "count": 4, + "success": 4, + "failure": 0, + "mean_latency_ms": 6.25, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 7, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 4, + "sum_ms": 25, + "max_ms": 7 + } + } + }, + "delivery": { + "queued": 7, + "completed": 9, + "pending": 0, + "processed_districts": 90, + "skipped_districts": 0, + "completion_mean_ms": 22.0, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 23, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 0, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 9, + "sum_ms": 198, + "max_ms": 23 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/coordinator/tpcc-1774910938757/summary.json b/modules/tpcc-results/coordinator/tpcc-1774910938757/summary.json new file mode 100644 index 00000000000..da574a497ae --- /dev/null +++ b/modules/tpcc-results/coordinator/tpcc-1774910938757/summary.json @@ -0,0 +1,348 @@ +{ + "run_id": "tpcc-1774910938757", + "driver_count": 1, + "drivers": [ + "driver-179375" + ], + "generated_at_ms": 1774911335944, + "total_transactions": 137, + "tpmc_like": 140.0, + "transaction_mix": { + "delivery": 5.839416058394161, + "new_order": 51.824817518248175, + "order_status": 4.37956204379562, + "payment": 34.306569343065696, + "stock_level": 3.6496350364963503 + }, + "conformance": { + "new_order_rollbacks": 1, + "new_order_total": 71, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 699, + "payment_remote": 0, + "payment_total": 47, + "payment_by_last_name": 28, + "order_status_by_last_name": 2, + "order_status_total": 6, + "delivery_queued": 8, + "delivery_completed": 13, + "delivery_processed_districts": 130, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 8, + "success": 8, + "failure": 0, + "mean_latency_ms": 7.875, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 9, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 8, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 8, + "sum_ms": 63, + "max_ms": 9 + } + }, + "new_order": { + "count": 71, + "success": 70, + "failure": 1, + "mean_latency_ms": 9.450704225352112, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 13, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 1, + 0, + 56, + 14, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 71, + "sum_ms": 671, + "max_ms": 13 + } + }, + "order_status": { + "count": 6, + "success": 6, + "failure": 0, + "mean_latency_ms": 1.3333333333333333, + "p50_latency_ms": 1, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 4, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 6, + "sum_ms": 8, + "max_ms": 2 + } + }, + "payment": { + "count": 47, + "success": 47, + "failure": 0, + "mean_latency_ms": 8.340425531914894, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 47, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 47, + "sum_ms": 392, + "max_ms": 10 + } + }, + "stock_level": { + "count": 5, + "success": 5, + "failure": 0, + "mean_latency_ms": 6.0, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 6, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 5, + "sum_ms": 30, + "max_ms": 6 + } + } + }, + "delivery": { + "queued": 8, + "completed": 13, + "pending": 0, + "processed_districts": 130, + "skipped_districts": 0, + "completion_mean_ms": 22.615384615384617, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 33, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 2, + 11, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 13, + "sum_ms": 294, + "max_ms": 33 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/coordinator/tpcc-1774911348130/summary.json b/modules/tpcc-results/coordinator/tpcc-1774911348130/summary.json new file mode 100644 index 00000000000..9de79650048 --- /dev/null +++ b/modules/tpcc-results/coordinator/tpcc-1774911348130/summary.json @@ -0,0 +1,348 @@ +{ + "run_id": "tpcc-1774911348130", + "driver_count": 1, + "drivers": [ + "driver-184437" + ], + "generated_at_ms": 1774911743197, + "total_transactions": 144, + "tpmc_like": 124.0, + "transaction_mix": { + "delivery": 4.166666666666667, + "new_order": 43.75, + "order_status": 6.25, + "payment": 43.75, + "stock_level": 2.0833333333333335 + }, + "conformance": { + "new_order_rollbacks": 1, + "new_order_total": 63, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 610, + "payment_remote": 0, + "payment_total": 63, + "payment_by_last_name": 35, + "order_status_by_last_name": 3, + "order_status_total": 9, + "delivery_queued": 6, + "delivery_completed": 7, + "delivery_processed_districts": 70, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 6, + "success": 6, + "failure": 0, + "mean_latency_ms": 8.0, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 9, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 6, + "sum_ms": 48, + "max_ms": 9 + } + }, + "new_order": { + "count": 63, + "success": 62, + "failure": 1, + "mean_latency_ms": 9.634920634920634, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 12, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 1, + 0, + 51, + 11, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 63, + "sum_ms": 607, + "max_ms": 12 + } + }, + "order_status": { + "count": 9, + "success": 9, + "failure": 0, + "mean_latency_ms": 1.2222222222222223, + "p50_latency_ms": 1, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 7, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 9, + "sum_ms": 11, + "max_ms": 2 + } + }, + "payment": { + "count": 63, + "success": 63, + "failure": 0, + "mean_latency_ms": 8.714285714285714, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 20, + "max_latency_ms": 11, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 62, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 63, + "sum_ms": 549, + "max_ms": 11 + } + }, + "stock_level": { + "count": 3, + "success": 3, + "failure": 0, + "mean_latency_ms": 5.666666666666667, + "p50_latency_ms": 5, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 7, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 2, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 3, + "sum_ms": 17, + "max_ms": 7 + } + } + }, + "delivery": { + "queued": 6, + "completed": 7, + "pending": 0, + "processed_districts": 70, + "skipped_districts": 0, + "completion_mean_ms": 21.714285714285715, + "completion_p50_ms": 20, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 32, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 4, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 152, + "max_ms": 32 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/tpcc-1774902157443/driver-110838/txn_events.ndjson b/modules/tpcc-results/tpcc-1774902157443/driver-110838/txn_events.ndjson new file mode 100644 index 00000000000..e69de29bb2d diff --git a/modules/tpcc-results/tpcc-1774908053099/driver-135613/summary.json b/modules/tpcc-results/tpcc-1774908053099/driver-135613/summary.json new file mode 100644 index 00000000000..cdc99d2238a --- /dev/null +++ b/modules/tpcc-results/tpcc-1774908053099/driver-135613/summary.json @@ -0,0 +1,354 @@ +{ + "run_id": "tpcc-1774908053099", + "driver_id": "driver-135613", + "uri": "http://127.0.0.1:3000", + "database": "tpcc-0", + "terminal_start": 1, + "terminals": 10, + "warehouse_count": 1, + "warmup_secs": 5, + "measure_secs": 30, + "measure_start_ms": 1774908117135, + "measure_end_ms": 1774908417135, + "generated_at_ms": 1774908449154, + "total_transactions": 129, + "tpmc_like": 134.0, + "transaction_mix": { + "delivery": 3.10077519379845, + "new_order": 51.93798449612403, + "order_status": 5.426356589147287, + "payment": 37.2093023255814, + "stock_level": 2.3255813953488373 + }, + "conformance": { + "new_order_rollbacks": 0, + "new_order_total": 67, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 649, + "payment_remote": 0, + "payment_total": 48, + "payment_by_last_name": 31, + "order_status_by_last_name": 3, + "order_status_total": 7, + "delivery_queued": 4, + "delivery_completed": 5, + "delivery_processed_districts": 50, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 4, + "success": 4, + "failure": 0, + "mean_latency_ms": 8.75, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 9, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 4, + "sum_ms": 35, + "max_ms": 9 + } + }, + "new_order": { + "count": 67, + "success": 67, + "failure": 0, + "mean_latency_ms": 9.701492537313433, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 11, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 54, + 13, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 67, + "sum_ms": 650, + "max_ms": 11 + } + }, + "order_status": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 1.7142857142857142, + "p50_latency_ms": 2, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 2, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 12, + "max_ms": 2 + } + }, + "payment": { + "count": 48, + "success": 48, + "failure": 0, + "mean_latency_ms": 8.458333333333334, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 48, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 48, + "sum_ms": 406, + "max_ms": 10 + } + }, + "stock_level": { + "count": 3, + "success": 3, + "failure": 0, + "mean_latency_ms": 5.666666666666667, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 6, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 1, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 3, + "sum_ms": 17, + "max_ms": 6 + } + } + }, + "delivery": { + "queued": 4, + "completed": 5, + "pending": 0, + "processed_districts": 50, + "skipped_districts": 0, + "completion_mean_ms": 24.8, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 40, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 1, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 5, + "sum_ms": 124, + "max_ms": 40 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/tpcc-1774908053099/driver-135613/txn_events.ndjson b/modules/tpcc-results/tpcc-1774908053099/driver-135613/txn_events.ndjson new file mode 100644 index 00000000000..d093f177fc4 --- /dev/null +++ b/modules/tpcc-results/tpcc-1774908053099/driver-135613/txn_events.ndjson @@ -0,0 +1,129 @@ +{"timestamp_ms":1774908121466,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908123028,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908124423,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908127571,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908128766,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908128813,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908130617,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908131168,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908132024,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908132322,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908133381,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908133904,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908136054,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908139719,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908140427,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908140680,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908142152,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908144719,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908145003,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908150378,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908150956,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908152727,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908160203,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908162820,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908163790,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908165653,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908167462,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908167470,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908167668,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908169378,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908169794,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908174548,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908182635,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908183780,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908186875,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908188634,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908190523,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908193785,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908197437,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908198027,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908198704,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908200820,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908206045,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908209758,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908213591,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908213883,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908214554,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908214560,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908215166,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908218653,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908221646,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908226323,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908228315,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908231465,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908231609,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908238965,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908241141,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908242514,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908245736,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908247835,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908249290,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908251095,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908251502,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908252595,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908254738,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908254986,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908255198,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"stock_level","success":true,"latency_ms":5,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908256398,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908256476,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908260433,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908260960,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908263528,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908263857,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908274343,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908277278,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908277947,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908278524,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908279834,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908282623,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908285219,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908286879,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908287687,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908289779,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908298806,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908304704,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908305322,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908306557,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908309276,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908310464,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908312748,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908315859,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908318039,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908319278,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908321339,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908322737,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908322956,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908332566,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908333067,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908339175,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908339942,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908344256,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908345147,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908345855,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908347891,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908352498,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908354355,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908354914,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908355858,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908364693,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908366305,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908369761,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908371345,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908372605,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908375644,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908377972,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908378750,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908380882,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908389858,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908393234,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908394970,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908397496,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908403370,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908404418,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908404676,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908406292,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908407090,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908408620,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":3,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908411669,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908416742,"run_id":"tpcc-1774908053099","driver_id":"driver-135613","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} diff --git a/modules/tpcc-results/tpcc-1774908671807/driver-137679/summary.json b/modules/tpcc-results/tpcc-1774908671807/driver-137679/summary.json new file mode 100644 index 00000000000..4b928b9fd8c --- /dev/null +++ b/modules/tpcc-results/tpcc-1774908671807/driver-137679/summary.json @@ -0,0 +1,354 @@ +{ + "run_id": "tpcc-1774908671807", + "driver_id": "driver-137679", + "uri": "http://127.0.0.1:3000", + "database": "tpcc-0", + "terminal_start": 1, + "terminals": 10, + "warehouse_count": 1, + "warmup_secs": 5, + "measure_secs": 30, + "measure_start_ms": 1774908738409, + "measure_end_ms": 1774909038409, + "generated_at_ms": 1774909101276, + "total_transactions": 151, + "tpmc_like": 138.0, + "transaction_mix": { + "delivery": 3.9735099337748343, + "new_order": 45.6953642384106, + "order_status": 4.635761589403973, + "payment": 41.05960264900662, + "stock_level": 4.635761589403973 + }, + "conformance": { + "new_order_rollbacks": 0, + "new_order_total": 69, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 735, + "payment_remote": 0, + "payment_total": 62, + "payment_by_last_name": 39, + "order_status_by_last_name": 4, + "order_status_total": 7, + "delivery_queued": 6, + "delivery_completed": 10, + "delivery_processed_districts": 100, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 6, + "success": 6, + "failure": 0, + "mean_latency_ms": 8.166666666666666, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 6, + "sum_ms": 49, + "max_ms": 10 + } + }, + "new_order": { + "count": 69, + "success": 69, + "failure": 0, + "mean_latency_ms": 9.91304347826087, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 16, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 51, + 18, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 69, + "sum_ms": 684, + "max_ms": 16 + } + }, + "order_status": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 1.1428571428571428, + "p50_latency_ms": 1, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 6, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 8, + "max_ms": 2 + } + }, + "payment": { + "count": 62, + "success": 62, + "failure": 0, + "mean_latency_ms": 8.661290322580646, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 20, + "max_latency_ms": 11, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 60, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 62, + "sum_ms": 537, + "max_ms": 11 + } + }, + "stock_level": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 5.428571428571429, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 6, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 3, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 38, + "max_ms": 6 + } + } + }, + "delivery": { + "queued": 6, + "completed": 10, + "pending": 0, + "processed_districts": 100, + "skipped_districts": 0, + "completion_mean_ms": 21.0, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 23, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 4, + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 10, + "sum_ms": 210, + "max_ms": 23 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/tpcc-1774908671807/driver-137679/txn_events.ndjson b/modules/tpcc-results/tpcc-1774908671807/driver-137679/txn_events.ndjson new file mode 100644 index 00000000000..7ed6c30ae41 --- /dev/null +++ b/modules/tpcc-results/tpcc-1774908671807/driver-137679/txn_events.ndjson @@ -0,0 +1,151 @@ +{"timestamp_ms":1774908741236,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908743144,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908745345,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908745536,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"payment","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908747970,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908750867,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908752176,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908753345,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908753762,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"stock_level","success":true,"latency_ms":5,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908754690,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908755680,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908756491,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908756824,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908757528,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908759914,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908761289,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908768639,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908769022,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908769271,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908771231,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908771729,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908772471,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908778051,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908778141,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908778285,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908788859,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908789941,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908794317,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908795015,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908797224,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908798877,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908799635,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908800468,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908801338,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908802675,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908809532,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"stock_level","success":true,"latency_ms":5,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908812586,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908815847,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908816410,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908817156,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908818314,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"payment","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908824321,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908824760,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908827202,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908830371,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908830900,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908831270,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908841063,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908844253,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908845238,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908847694,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908848742,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908850201,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908856059,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":16,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908856397,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908856988,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908859195,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908859849,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908860382,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908861753,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908862375,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908865409,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908865911,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908868093,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908870806,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908877438,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908878128,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908878643,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908886023,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908886748,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908888063,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908888124,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908892497,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908896306,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908896525,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908897237,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908897929,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908898905,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908901547,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908901594,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908902030,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908905548,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908906349,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908913260,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908917908,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908922766,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908925875,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908926201,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908926731,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908928442,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908928749,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908928979,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908929587,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908931036,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908931723,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908934351,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908938128,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908939870,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"delivery","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908940272,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908942231,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908949939,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908952038,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"delivery","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908953561,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908954601,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908955039,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908959635,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908960337,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908961355,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908962292,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908962880,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908964628,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908965602,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908969654,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908970138,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908970170,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908971268,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908974579,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908979707,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908981536,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908981735,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908983018,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908984912,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908985963,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908988567,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908989507,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908994028,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908997021,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908997286,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774908998737,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909000134,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909000799,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909000822,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909004720,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909006840,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909007299,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909011923,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909014143,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909014650,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"delivery","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909015718,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909017990,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909018446,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909020104,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909024175,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909027737,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909031245,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909032175,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":5,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909032611,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":2,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909033487,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909034569,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909036146,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909037046,"run_id":"tpcc-1774908671807","driver_id":"driver-137679","terminal_id":10,"transaction":"stock_level","success":true,"latency_ms":4,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} diff --git a/modules/tpcc-results/tpcc-1774909172568/driver-138825/txn_events.ndjson b/modules/tpcc-results/tpcc-1774909172568/driver-138825/txn_events.ndjson new file mode 100644 index 00000000000..fabc60b29ac --- /dev/null +++ b/modules/tpcc-results/tpcc-1774909172568/driver-138825/txn_events.ndjson @@ -0,0 +1,30 @@ +{"timestamp_ms":1774909244308,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909248022,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909251460,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":1,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909258885,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909262108,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909262639,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909264356,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909266079,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909266363,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909267213,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909271161,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":3,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909278965,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":6,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909281550,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909285071,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909285455,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909288286,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909288804,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909289708,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":1,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909290317,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909290947,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909291594,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909293561,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909294141,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909295389,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909298121,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":1,"transaction":"stock_level","success":true,"latency_ms":5,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909300404,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909301778,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909303514,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909304011,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909311056,"run_id":"tpcc-1774909172568","driver_id":"driver-138825","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":8," \ No newline at end of file diff --git a/modules/tpcc-results/tpcc-1774909349292/driver-152145/summary.json b/modules/tpcc-results/tpcc-1774909349292/driver-152145/summary.json new file mode 100644 index 00000000000..8422823beeb --- /dev/null +++ b/modules/tpcc-results/tpcc-1774909349292/driver-152145/summary.json @@ -0,0 +1,354 @@ +{ + "run_id": "tpcc-1774909349292", + "driver_id": "driver-152145", + "uri": "http://127.0.0.1:3000", + "database": "tpcc-0", + "terminal_start": 1, + "terminals": 10, + "warehouse_count": 1, + "warmup_secs": 5, + "measure_secs": 30, + "measure_start_ms": 1774909414551, + "measure_end_ms": 1774909714551, + "generated_at_ms": 1774909758803, + "total_transactions": 143, + "tpmc_like": 132.0, + "transaction_mix": { + "delivery": 4.895104895104895, + "new_order": 48.25174825174825, + "order_status": 1.3986013986013985, + "payment": 42.65734265734266, + "stock_level": 2.797202797202797 + }, + "conformance": { + "new_order_rollbacks": 3, + "new_order_total": 69, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 719, + "payment_remote": 0, + "payment_total": 61, + "payment_by_last_name": 35, + "order_status_by_last_name": 0, + "order_status_total": 2, + "delivery_queued": 7, + "delivery_completed": 9, + "delivery_processed_districts": 90, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 7, + "success": 7, + "failure": 0, + "mean_latency_ms": 7.857142857142857, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 55, + "max_ms": 10 + } + }, + "new_order": { + "count": 69, + "success": 66, + "failure": 3, + "mean_latency_ms": 9.782608695652174, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 19, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 1, + 2, + 0, + 43, + 23, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 69, + "sum_ms": 675, + "max_ms": 19 + } + }, + "order_status": { + "count": 2, + "success": 2, + "failure": 0, + "mean_latency_ms": 1.0, + "p50_latency_ms": 1, + "p95_latency_ms": 1, + "p99_latency_ms": 1, + "max_latency_ms": 1, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 2, + "sum_ms": 2, + "max_ms": 1 + } + }, + "payment": { + "count": 61, + "success": 61, + "failure": 0, + "mean_latency_ms": 8.721311475409836, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 20, + "max_latency_ms": 14, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 59, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 61, + "sum_ms": 532, + "max_ms": 14 + } + }, + "stock_level": { + "count": 4, + "success": 4, + "failure": 0, + "mean_latency_ms": 6.25, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 7, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 4, + "sum_ms": 25, + "max_ms": 7 + } + } + }, + "delivery": { + "queued": 7, + "completed": 9, + "pending": 0, + "processed_districts": 90, + "skipped_districts": 0, + "completion_mean_ms": 22.0, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 23, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 0, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 9, + "sum_ms": 198, + "max_ms": 23 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/tpcc-1774909349292/driver-152145/txn_events.ndjson b/modules/tpcc-results/tpcc-1774909349292/driver-152145/txn_events.ndjson new file mode 100644 index 00000000000..9ef507b9a5f --- /dev/null +++ b/modules/tpcc-results/tpcc-1774909349292/driver-152145/txn_events.ndjson @@ -0,0 +1,143 @@ +{"timestamp_ms":1774909415523,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909418015,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909418318,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909419652,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909420772,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909421012,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909422177,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909427669,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909430506,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":false,"latency_ms":1,"rollback":true,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":"item 100001 not found"} +{"timestamp_ms":1774909432494,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909434140,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909435419,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909445283,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909449324,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909450087,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909461615,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"new_order","success":false,"latency_ms":2,"rollback":true,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":"item 100001 not found"} +{"timestamp_ms":1774909463025,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909465626,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909468921,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909469422,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909469686,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909470109,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"delivery","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909471730,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909472641,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":14,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909472899,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909473798,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909475135,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909477754,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909478983,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909482708,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909482905,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909482951,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909484044,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909492086,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909494390,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909497407,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"new_order","success":false,"latency_ms":2,"rollback":true,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":"item 100001 not found"} +{"timestamp_ms":1774909498785,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909502205,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909507094,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909507246,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909507730,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909511157,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909514429,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909516172,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909517706,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909517924,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909518275,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909520232,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909520649,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909521985,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909529395,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909529606,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909533982,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909536868,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909537552,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909538036,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909540125,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909540514,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909541636,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909542845,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909549397,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909552349,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909553488,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909557586,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909558968,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909562077,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909563462,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909563946,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909566524,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909568607,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909570458,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909573340,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909574191,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909574360,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909577009,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909577725,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909585014,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909588317,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909589303,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909593418,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909595951,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909596155,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"stock_level","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909599589,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909601059,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909603655,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909603877,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909604815,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909604912,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909610750,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909612972,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909613428,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909614081,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909615898,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909617364,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909617661,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909619216,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909619710,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909621244,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909623584,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909625248,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909625607,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909630115,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909631421,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909634407,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909635758,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909635883,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909637581,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909641657,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909642151,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909644231,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"delivery","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909644667,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"delivery","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909646383,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909647273,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909651051,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"payment","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909652976,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909653296,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909653384,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909654808,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909655136,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909657243,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909658694,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909659080,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909662020,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909662063,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909671503,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":19,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909675387,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909676616,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909676783,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909677417,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909684358,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909684919,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909688030,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909689112,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909691867,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909694450,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909697472,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909698081,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909702423,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909703024,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909708654,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909710370,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909712918,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774909713070,"run_id":"tpcc-1774909349292","driver_id":"driver-152145","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} diff --git a/modules/tpcc-results/tpcc-1774910938757/driver-179375/summary.json b/modules/tpcc-results/tpcc-1774910938757/driver-179375/summary.json new file mode 100644 index 00000000000..31203cab4cb --- /dev/null +++ b/modules/tpcc-results/tpcc-1774910938757/driver-179375/summary.json @@ -0,0 +1,354 @@ +{ + "run_id": "tpcc-1774910938757", + "driver_id": "driver-179375", + "uri": "http://127.0.0.1:3000", + "database": "tpcc-0", + "terminal_start": 1, + "terminals": 10, + "warehouse_count": 1, + "warmup_secs": 5, + "measure_secs": 30, + "measure_start_ms": 1774911013905, + "measure_end_ms": 1774911313905, + "generated_at_ms": 1774911335911, + "total_transactions": 137, + "tpmc_like": 140.0, + "transaction_mix": { + "delivery": 5.839416058394161, + "new_order": 51.824817518248175, + "order_status": 4.37956204379562, + "payment": 34.306569343065696, + "stock_level": 3.6496350364963503 + }, + "conformance": { + "new_order_rollbacks": 1, + "new_order_total": 71, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 699, + "payment_remote": 0, + "payment_total": 47, + "payment_by_last_name": 28, + "order_status_by_last_name": 2, + "order_status_total": 6, + "delivery_queued": 8, + "delivery_completed": 13, + "delivery_processed_districts": 130, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 8, + "success": 8, + "failure": 0, + "mean_latency_ms": 7.875, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 9, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 8, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 8, + "sum_ms": 63, + "max_ms": 9 + } + }, + "new_order": { + "count": 71, + "success": 70, + "failure": 1, + "mean_latency_ms": 9.450704225352112, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 13, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 1, + 0, + 56, + 14, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 71, + "sum_ms": 671, + "max_ms": 13 + } + }, + "order_status": { + "count": 6, + "success": 6, + "failure": 0, + "mean_latency_ms": 1.3333333333333333, + "p50_latency_ms": 1, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 4, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 6, + "sum_ms": 8, + "max_ms": 2 + } + }, + "payment": { + "count": 47, + "success": 47, + "failure": 0, + "mean_latency_ms": 8.340425531914894, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 10, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 47, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 47, + "sum_ms": 392, + "max_ms": 10 + } + }, + "stock_level": { + "count": 5, + "success": 5, + "failure": 0, + "mean_latency_ms": 6.0, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 6, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 5, + "sum_ms": 30, + "max_ms": 6 + } + } + }, + "delivery": { + "queued": 8, + "completed": 13, + "pending": 0, + "processed_districts": 130, + "skipped_districts": 0, + "completion_mean_ms": 22.615384615384617, + "completion_p50_ms": 50, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 33, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 2, + 11, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 13, + "sum_ms": 294, + "max_ms": 33 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/tpcc-1774910938757/driver-179375/txn_events.ndjson b/modules/tpcc-results/tpcc-1774910938757/driver-179375/txn_events.ndjson new file mode 100644 index 00000000000..1efcb34c1b1 --- /dev/null +++ b/modules/tpcc-results/tpcc-1774910938757/driver-179375/txn_events.ndjson @@ -0,0 +1,137 @@ +{"timestamp_ms":1774911015678,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"delivery","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911018420,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911018981,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911021714,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911025655,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911028533,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911029620,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911030037,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911030687,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911031628,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911035974,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911037311,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911038794,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911041094,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911041533,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911041838,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911043746,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911045138,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911046003,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911048103,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911049036,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911058201,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911060595,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911063231,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911066017,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911066719,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911066734,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911071474,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911072336,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911073570,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911080582,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911081056,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911082601,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911086211,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911087124,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911092334,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911093279,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911099642,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911100108,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911104086,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911112640,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911116118,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911116537,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911117263,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911117646,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911118944,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911120695,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911123677,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911125237,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911126810,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911128147,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911130474,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911136200,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911136601,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911140072,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911141671,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911144629,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911144689,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911147387,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911147435,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911149874,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911150002,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911150500,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911153549,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911155410,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911157451,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911158883,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911160945,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911165938,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911166260,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911168596,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911168982,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911172632,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911179945,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911180341,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911180750,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911180893,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911182413,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"delivery","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911183739,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911185140,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911186938,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911188187,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911191634,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911193115,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":13,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911196476,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911196640,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911197762,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911202852,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911207014,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911209506,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911212317,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911215158,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911215179,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911216257,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911219060,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911228010,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911228606,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911231138,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911235317,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911236516,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911238479,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911248549,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911248750,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911251552,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911252007,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911253797,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911255504,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":3,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911257262,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911258409,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911261044,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911264120,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911264348,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911264884,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911266466,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911268976,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911269608,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911271250,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911272908,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911273183,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911273789,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911278341,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911279930,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911282084,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911287065,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911295995,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911297359,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911297442,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911298895,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911300893,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"stock_level","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911301946,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911303064,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":2,"transaction":"new_order","success":false,"latency_ms":2,"rollback":true,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":"item 100001 not found"} +{"timestamp_ms":1774911307590,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911307662,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911308510,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911311078,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911312776,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":4,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911313398,"run_id":"tpcc-1774910938757","driver_id":"driver-179375","terminal_id":7,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} diff --git a/modules/tpcc-results/tpcc-1774911348130/driver-184437/summary.json b/modules/tpcc-results/tpcc-1774911348130/driver-184437/summary.json new file mode 100644 index 00000000000..c9d27156bec --- /dev/null +++ b/modules/tpcc-results/tpcc-1774911348130/driver-184437/summary.json @@ -0,0 +1,354 @@ +{ + "run_id": "tpcc-1774911348130", + "driver_id": "driver-184437", + "uri": "http://127.0.0.1:3000", + "database": "tpcc-0", + "terminal_start": 1, + "terminals": 10, + "warehouse_count": 1, + "warmup_secs": 5, + "measure_secs": 30, + "measure_start_ms": 1774911415116, + "measure_end_ms": 1774911715116, + "generated_at_ms": 1774911743163, + "total_transactions": 144, + "tpmc_like": 124.0, + "transaction_mix": { + "delivery": 4.166666666666667, + "new_order": 43.75, + "order_status": 6.25, + "payment": 43.75, + "stock_level": 2.0833333333333335 + }, + "conformance": { + "new_order_rollbacks": 1, + "new_order_total": 63, + "new_order_remote_order_lines": 0, + "new_order_total_order_lines": 610, + "payment_remote": 0, + "payment_total": 63, + "payment_by_last_name": 35, + "order_status_by_last_name": 3, + "order_status_total": 9, + "delivery_queued": 6, + "delivery_completed": 7, + "delivery_processed_districts": 70, + "delivery_skipped_districts": 0 + }, + "transactions": { + "delivery": { + "count": 6, + "success": 6, + "failure": 0, + "mean_latency_ms": 8.0, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 9, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 6, + "sum_ms": 48, + "max_ms": 9 + } + }, + "new_order": { + "count": 63, + "success": 62, + "failure": 1, + "mean_latency_ms": 9.634920634920634, + "p50_latency_ms": 10, + "p95_latency_ms": 20, + "p99_latency_ms": 20, + "max_latency_ms": 12, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 1, + 0, + 51, + 11, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 63, + "sum_ms": 607, + "max_ms": 12 + } + }, + "order_status": { + "count": 9, + "success": 9, + "failure": 0, + "mean_latency_ms": 1.2222222222222223, + "p50_latency_ms": 1, + "p95_latency_ms": 2, + "p99_latency_ms": 2, + "max_latency_ms": 2, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 7, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 9, + "sum_ms": 11, + "max_ms": 2 + } + }, + "payment": { + "count": 63, + "success": 63, + "failure": 0, + "mean_latency_ms": 8.714285714285714, + "p50_latency_ms": 10, + "p95_latency_ms": 10, + "p99_latency_ms": 20, + "max_latency_ms": 11, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 62, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 63, + "sum_ms": 549, + "max_ms": 11 + } + }, + "stock_level": { + "count": 3, + "success": 3, + "failure": 0, + "mean_latency_ms": 5.666666666666667, + "p50_latency_ms": 5, + "p95_latency_ms": 10, + "p99_latency_ms": 10, + "max_latency_ms": 7, + "histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 2, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 3, + "sum_ms": 17, + "max_ms": 7 + } + } + }, + "delivery": { + "queued": 6, + "completed": 7, + "pending": 0, + "processed_districts": 70, + "skipped_districts": 0, + "completion_mean_ms": 21.714285714285715, + "completion_p50_ms": 20, + "completion_p95_ms": 50, + "completion_p99_ms": 50, + "completion_max_ms": 32, + "completion_histogram": { + "buckets_ms": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + 60000, + 120000 + ], + "counts": [ + 0, + 0, + 0, + 0, + 4, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "count": 7, + "sum_ms": 152, + "max_ms": 32 + } + } +} \ No newline at end of file diff --git a/modules/tpcc-results/tpcc-1774911348130/driver-184437/txn_events.ndjson b/modules/tpcc-results/tpcc-1774911348130/driver-184437/txn_events.ndjson new file mode 100644 index 00000000000..b77e826e899 --- /dev/null +++ b/modules/tpcc-results/tpcc-1774911348130/driver-184437/txn_events.ndjson @@ -0,0 +1,144 @@ +{"timestamp_ms":1774911415270,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911415875,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911416024,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911420011,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911420564,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911420614,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911422674,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911423192,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911426327,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911426598,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911427420,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911430162,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911431498,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911432373,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911433687,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911435747,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911436663,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911440674,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911441706,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911445613,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911452070,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911452086,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":6,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911454872,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":12,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911455746,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911458933,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911459003,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911459127,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911461957,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911464188,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911469261,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911469447,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911472722,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911473037,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"stock_level","success":true,"latency_ms":5,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911473300,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":7,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911474316,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911475259,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911478946,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911479381,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911482284,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911484614,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911486628,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911487926,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911491138,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911491625,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911493935,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911494700,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911508104,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911509902,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911510757,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911515518,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911518298,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911518616,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911520785,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911522800,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911524204,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911524385,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911525364,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911527729,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911530253,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911531032,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911531531,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911531755,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911535161,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911535191,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911536771,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911538960,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911551268,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911551506,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911553296,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911557816,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911563311,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911568854,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911569428,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911569630,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911572917,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911575050,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911575619,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"new_order","success":false,"latency_ms":2,"rollback":true,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":"item 100001 not found"} +{"timestamp_ms":1774911577565,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911578145,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911578312,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911578420,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"stock_level","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911583673,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911586412,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911589100,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911593279,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911593880,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911595515,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911596638,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911598090,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"stock_level","success":true,"latency_ms":5,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911599396,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911604078,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911604774,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911606709,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911609960,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":11,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911611447,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911612429,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911613753,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911616178,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911617410,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911617812,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911619000,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911619173,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"delivery","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911622639,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911622866,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911628352,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911632272,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":15,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911633764,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911635699,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911637262,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":12,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911637912,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911638099,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911640305,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911642474,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911642832,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911645763,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":13,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911648083,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"delivery","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911650708,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911653037,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":9,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":10,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911655222,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911655965,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911659030,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911661467,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911662554,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":14,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911666256,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911668107,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911668216,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911668705,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911671445,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911682394,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":5,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911683827,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":2,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911686799,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":1,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911687134,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911689308,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":7,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":6,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911690291,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":11,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911691615,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"new_order","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":8,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911692446,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":10,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":5,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911694741,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":4,"transaction":"new_order","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":9,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911695095,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"delivery","success":true,"latency_ms":7,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911698029,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":3,"transaction":"payment","success":true,"latency_ms":10,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911702984,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"order_status","success":true,"latency_ms":1,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911705406,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"order_status","success":true,"latency_ms":2,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911709380,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":true,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911711415,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":6,"transaction":"payment","success":true,"latency_ms":9,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} +{"timestamp_ms":1774911713374,"run_id":"tpcc-1774911348130","driver_id":"driver-184437","terminal_id":8,"transaction":"payment","success":true,"latency_ms":8,"rollback":false,"remote":false,"by_last_name":false,"order_line_count":0,"remote_order_line_count":0,"detail":null} diff --git a/modules/tpcc/src/lib.rs b/modules/tpcc/src/lib.rs index d1e882effd4..e62ed165d64 100644 --- a/modules/tpcc/src/lib.rs +++ b/modules/tpcc/src/lib.rs @@ -487,12 +487,7 @@ pub fn order_status( } #[reducer] -pub fn stock_level( - ctx: &ReducerContext, - w_id: u32, - d_id: u8, - threshold: i32, -) -> Result { +pub fn stock_level(ctx: &ReducerContext, w_id: u32, d_id: u8, threshold: i32) -> Result { let _timer = LogStopwatch::new("stock_level"); let _timer_district = LogStopwatch::new("stock_level_district"); @@ -514,7 +509,7 @@ pub fn stock_level( _timer_filter.end(); let mut low_stock_count = 0u32; - let _timer_count= LogStopwatch::new("stock_level_count"); + let _timer_count = LogStopwatch::new("stock_level_count"); for item_id in item_ids { if find_low_stock(ctx, w_id, item_id, threshold).is_some() { low_stock_count += 1; diff --git a/modules/tpcc/src/load.rs b/modules/tpcc/src/load.rs index 72fa21f3866..9b0437b4dbe 100644 --- a/modules/tpcc/src/load.rs +++ b/modules/tpcc/src/load.rs @@ -53,6 +53,8 @@ pub struct TpccLoadConfigRequest { pub database_number: u32, pub num_databases: u32, pub warehouses_per_database: u32, + pub warehouse_id_offset: u32, + pub skip_items: bool, pub batch_size: u32, pub seed: u64, pub load_c_last: u32, @@ -69,6 +71,8 @@ pub struct TpccLoadConfig { pub database_number: u32, pub num_databases: u32, pub warehouses_per_database: u32, + pub warehouse_id_offset: u32, + pub skip_items: bool, pub batch_size: u32, pub seed: u64, pub load_c_last: u32, @@ -227,6 +231,8 @@ fn configure_tpcc_load_internal(ctx: &ReducerContext, request: TpccLoadConfigReq database_number: request.database_number, num_databases: request.num_databases, warehouses_per_database: request.warehouses_per_database, + warehouse_id_offset: request.warehouse_id_offset, + skip_items: request.skip_items, batch_size: request.batch_size, seed: request.seed, load_c_last: request.load_c_last, @@ -265,6 +271,21 @@ fn validate_request(request: &TpccLoadConfigRequest) -> Result<(), String> { request.num_databases, request.warehouses_per_database )); } + // Validate that the warehouse ID range for this database doesn't overflow u32. + // warehouse_start = database_number * warehouses_per_database + warehouse_id_offset + 1 + // warehouse_end = warehouse_start + warehouses_per_database - 1 + if request + .database_number + .checked_mul(request.warehouses_per_database) + .and_then(|v| v.checked_add(request.warehouse_id_offset)) + .and_then(|v| v.checked_add(request.warehouses_per_database)) + .is_none() + { + return Err(format!( + "warehouse id range overflow u32 (database_number={}, warehouses_per_database={}, warehouse_id_offset={})", + request.database_number, request.warehouses_per_database, request.warehouse_id_offset + )); + } Ok(()) } @@ -272,8 +293,16 @@ fn initial_state(request: &TpccLoadConfigRequest, now: Timestamp) -> TpccLoadSta TpccLoadState { singleton_id: LOAD_SINGLETON_ID, status: TpccLoadStatus::Idle, - phase: TpccLoadPhase::Items, - next_warehouse_id: warehouse_start(request.database_number, request.warehouses_per_database), + phase: if request.skip_items { + TpccLoadPhase::WarehousesDistricts + } else { + TpccLoadPhase::Items + }, + next_warehouse_id: warehouse_start( + request.database_number, + request.warehouses_per_database, + request.warehouse_id_offset, + ), next_district_id: 1, next_item_id: 1, next_order_id: 1, @@ -291,6 +320,8 @@ fn config_as_request(config: &TpccLoadConfig) -> TpccLoadConfigRequest { database_number: config.database_number, num_databases: config.num_databases, warehouses_per_database: config.warehouses_per_database, + warehouse_id_offset: config.warehouse_id_offset, + skip_items: config.skip_items, batch_size: config.batch_size, seed: config.seed, load_c_last: config.load_c_last, @@ -306,8 +337,9 @@ fn build_remote_warehouses(request: &TpccLoadConfigRequest) -> Vec Result { let _timer = LogStopwatch::new("load_warehouses_district_chunk"); - let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database); - if job.next_warehouse_id < warehouse_start(config.database_number, config.warehouses_per_database) + let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database, config.warehouse_id_offset); + if job.next_warehouse_id < warehouse_start(config.database_number, config.warehouses_per_database, config.warehouse_id_offset) || job.next_warehouse_id > end_warehouse { return Err(format!("invalid warehouse cursor {}", job.next_warehouse_id)); @@ -436,7 +468,7 @@ fn load_warehouse_district_chunk( TpccLoadPhase::WarehousesDistricts }, next_warehouse_id: if job.next_warehouse_id == end_warehouse { - warehouse_start(config.database_number, config.warehouses_per_database) + warehouse_start(config.database_number, config.warehouses_per_database, config.warehouse_id_offset) } else { job.next_warehouse_id + 1 }, @@ -450,8 +482,8 @@ fn load_warehouse_district_chunk( fn load_stock_chunk(ctx: &ReducerContext, config: &TpccLoadConfig, job: &TpccLoadJob) -> Result { let _timer = LogStopwatch::new("load_stock_chunk"); - let start_warehouse = warehouse_start(config.database_number, config.warehouses_per_database); - let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database); + let start_warehouse = warehouse_start(config.database_number, config.warehouses_per_database, config.warehouse_id_offset); + let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database, config.warehouse_id_offset); if job.next_warehouse_id < start_warehouse || job.next_warehouse_id > end_warehouse { return Err(format!("invalid stock warehouse cursor {}", job.next_warehouse_id)); } @@ -491,8 +523,8 @@ fn load_customer_history_chunk( job: &TpccLoadJob, ) -> Result { let _timer = LogStopwatch::new("load_customer_history_chunk"); - let start_warehouse = warehouse_start(config.database_number, config.warehouses_per_database); - let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database); + let start_warehouse = warehouse_start(config.database_number, config.warehouses_per_database, config.warehouse_id_offset); + let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database, config.warehouse_id_offset); if job.next_warehouse_id < start_warehouse || job.next_warehouse_id > end_warehouse { return Err(format!("invalid customer warehouse cursor {}", job.next_warehouse_id)); } @@ -541,8 +573,8 @@ fn load_customer_history_chunk( fn load_order_chunk(ctx: &ReducerContext, config: &TpccLoadConfig, job: &TpccLoadJob) -> Result { let _timer = LogStopwatch::new("load_order_chunk"); - let start_warehouse = warehouse_start(config.database_number, config.warehouses_per_database); - let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database); + let start_warehouse = warehouse_start(config.database_number, config.warehouses_per_database, config.warehouse_id_offset); + let end_warehouse = warehouse_end(config.database_number, config.warehouses_per_database, config.warehouse_id_offset); if job.next_warehouse_id < start_warehouse || job.next_warehouse_id > end_warehouse { return Err(format!("invalid order warehouse cursor {}", job.next_warehouse_id)); } @@ -834,21 +866,26 @@ fn customer_permutation(config: &TpccLoadConfig, warehouse_id: WarehouseId, dist permutation } -fn warehouse_range(database_number: u32, warehouses_per_database: u32) -> std::ops::Range { - let start = warehouse_start(database_number, warehouses_per_database); +fn warehouse_range( + database_number: u32, + warehouses_per_database: u32, + offset: u32, +) -> std::ops::Range { + let start = warehouse_start(database_number, warehouses_per_database, offset); let end = start + warehouses_per_database; start..end } -fn warehouse_start(database_number: u32, warehouses_per_database: u32) -> WarehouseId { +fn warehouse_start(database_number: u32, warehouses_per_database: u32, offset: u32) -> WarehouseId { database_number .checked_mul(warehouses_per_database) + .and_then(|value| value.checked_add(offset)) .and_then(|value| value.checked_add(1)) .expect("warehouse id arithmetic validated at configure_tpcc_load time") } -fn warehouse_end(database_number: u32, warehouses_per_database: u32) -> WarehouseId { - warehouse_start(database_number, warehouses_per_database) + warehouses_per_database - 1 +fn warehouse_end(database_number: u32, warehouses_per_database: u32, offset: u32) -> WarehouseId { + warehouse_start(database_number, warehouses_per_database, offset) + warehouses_per_database - 1 } fn deterministic_rng(seed: u64, tag: u64, parts: &[u64]) -> StdRng { diff --git a/modules/tpcc/src/new_order.rs b/modules/tpcc/src/new_order.rs index 7fd0c939b04..00dc06fa434 100644 --- a/modules/tpcc/src/new_order.rs +++ b/modules/tpcc/src/new_order.rs @@ -1,9 +1,7 @@ use std::collections::HashMap; use crate::{ - district, find_customer_by_id, find_district, find_stock, find_warehouse, item, order_line, pack_order_key, - remote::{call_remote_reducer, remote_warehouse_home}, - stock, District, Item, OrderLine, Stock, WarehouseId, DISTRICTS_PER_WAREHOUSE, TAX_SCALE, + DISTRICTS_PER_WAREHOUSE, District, Item, OrderLine, Stock, TAX_SCALE, WarehouseId, district, find_customer_by_id, find_district, find_stock, find_warehouse, item, order_line, pack_order_key, remote::{call_remote_reducer, remote_warehouse_home, simulate_remote_call}, stock }; use spacetimedb::{log_stopwatch::LogStopwatch, reducer, Identity, ReducerContext, SpacetimeType, Table, Timestamp}; @@ -189,6 +187,13 @@ fn call_remote_order_multiple_items_and_decrement_stock( "order_multiple_items_and_decrement_stocks", &input, ) + // simulate_remote_call( + // ctx, + // remote_database_identity, + // "order_multiple_items_and_decrement_stocks", + // &input, + // )?; + // Ok(simulated_remote_order_outputs(input)) } struct ProcessedNewOrderItem { @@ -262,6 +267,29 @@ pub struct OrderMultipleItemsInput { terminal_warehouse: WarehouseId, } +fn simulated_remote_order_outputs(input: OrderMultipleItemsInput) -> Vec { + input + .lines + .into_iter() + .map(|line| simulated_remote_order_output(line, input.district)) + .collect() +} + +fn simulated_remote_order_output(line: NewOrderLineAndIndex, district: u8) -> OrderItemOutput { + let stock_data = if line.line.item_id % 10 == 0 { + "SIMULATED ORIGINAL STOCK" + } else { + "SIMULATED REMOTE STOCK" + }; + + OrderItemOutput { + s_dist: format!("REMOTE-D{district:02}"), + s_data: stock_data.to_string(), + updated_quantity: adjust_stock_quantity(100, line.line.quantity as i32), + index: line.index, + } +} + #[reducer] fn order_multiple_items_and_decrement_stocks( ctx: &ReducerContext, diff --git a/modules/tpcc/src/payment.rs b/modules/tpcc/src/payment.rs index 59710db9587..c64e533abcd 100644 --- a/modules/tpcc/src/payment.rs +++ b/modules/tpcc/src/payment.rs @@ -1,7 +1,5 @@ use crate::{ - customer, district, find_district, find_warehouse, history, - remote::{call_remote_reducer, remote_warehouse_home}, - resolve_customer, warehouse, Customer, CustomerSelector, District, History, Warehouse, MAX_C_DATA_LEN, + Customer, CustomerSelector, District, History, MAX_C_DATA_LEN, Warehouse, customer, district, find_district, find_warehouse, history, remote::{call_remote_reducer, remote_warehouse_home, simulate_remote_call}, resolve_customer, warehouse }; use spacetimedb::{ log_stopwatch::LogStopwatch, procedure, reducer, Identity, ProcedureContext, ReducerContext, SpacetimeType, Table, @@ -134,6 +132,13 @@ fn call_remote_resolve_and_update_customer_for_payment( "resolve_and_update_customer_for_payment", request, ) + // simulate_remote_call( + // ctx, + // remote_database_identity, + // "resolve_and_update_customer_for_payment", + // request, + // )?; + // Ok(simulated_remote_customer(request)) } #[procedure] @@ -149,6 +154,53 @@ fn process_remote_payment(ctx: &mut ProcedureContext, request: PaymentRequest) - }) } +fn simulated_remote_customer(request: &PaymentRequest) -> Customer { + let (customer_id, customer_first, customer_last) = match &request.customer_selector { + CustomerSelector::ById(customer_id) => ( + *customer_id, + format!("Remote{customer_id}"), + format!("Customer{customer_id}"), + ), + CustomerSelector::ByLastName(last_name) => ( + simulated_customer_id_from_last_name(last_name), + "Remote".to_string(), + last_name.clone(), + ), + }; + + Customer { + customer_key: 0, + c_w_id: request.customer_warehouse_id, + c_d_id: request.customer_district_id, + c_id: customer_id, + c_first: customer_first, + c_middle: "OE".to_string(), + c_last: customer_last, + c_street_1: "REMOTE".to_string(), + c_street_2: "SIMULATED".to_string(), + c_city: "REMOTE".to_string(), + c_state: "RM".to_string(), + c_zip: "000000000".to_string(), + c_phone: "0000000000000000".to_string(), + c_since: request.now, + c_credit: "GC".to_string(), + c_credit_lim_cents: 5_000_000, + c_discount_bps: 0, + c_balance_cents: -request.payment_amount_cents, + c_ytd_payment_cents: request.payment_amount_cents, + c_payment_cnt: 1, + c_delivery_cnt: 0, + c_data: "SIMULATED_REMOTE_PAYMENT".to_string(), + } +} + +fn simulated_customer_id_from_last_name(last_name: &str) -> u32 { + let hash = last_name + .bytes() + .fold(0_u32, |acc, byte| acc.wrapping_mul(31).wrapping_add(u32::from(byte))); + (hash % crate::CUSTOMERS_PER_DISTRICT) + 1 +} + fn update_customer(tx: &ReducerContext, request: &PaymentRequest, customer: Customer) -> Customer { let mut updated_customer = Customer { c_balance_cents: customer.c_balance_cents - request.payment_amount_cents, diff --git a/modules/tpcc/src/remote.rs b/modules/tpcc/src/remote.rs index addd526d150..79f7404e8ae 100644 --- a/modules/tpcc/src/remote.rs +++ b/modules/tpcc/src/remote.rs @@ -1,10 +1,11 @@ use spacetimedb::{ - reducer, - remote_reducer::{call_reducer_on_db, into_reducer_error_message, RemoteCallError}, - table, DeserializeOwned, Identity, ReducerContext, Serialize, SpacetimeType, Table, + reducer, remote_reducer::call_reducer_on_db_2pc, table, DeserializeOwned, Identity, ReducerContext, Serialize, + SpacetimeType, Table, }; use spacetimedb_sats::bsatn; +const SIMULATED_REMOTE_CALL_WORK: u32 = 1_000_000; + /// For warehouses not managed by this database, stores the [`Identity`] of the remote database which manages that warehouse. /// /// Will not have a row present for a warehouse managed by the local database. @@ -42,6 +43,40 @@ pub fn remote_warehouse_home(ctx: &ReducerContext, warehouse_id: u32) -> Option< .map(|row| row.remote_database_home) } +pub fn simulate_remote_call( + _ctx: &ReducerContext, + database_ident: Identity, + reducer_name: &str, + args: &Args, +) -> Result<(), String> +where + Args: SpacetimeType + Serialize, +{ + let args = bsatn::to_vec(args).map_err(|e| { + format!( + "Failed to BSATN-serialize args for simulated remote reducer {reducer_name} on database {database_ident}: {e}" + ) + })?; + + // Hold the reducer busy to approximate a synchronous remote call while we debug the real remote-call path. + let mut state = mix_remote_work(0x9E37_79B9_7F4A_7C15 ^ u64::try_from(args.len()).unwrap_or(u64::MAX)); + for byte in format!("{database_ident}:{reducer_name}").bytes() { + state = mix_remote_work(state ^ u64::from(byte)); + } + for &byte in &args { + state = mix_remote_work(state ^ u64::from(byte)); + } + + let rounds = + SIMULATED_REMOTE_CALL_WORK.saturating_add(u32::try_from(args.len()).unwrap_or(u32::MAX).saturating_mul(128)); + for _ in 0..rounds { + state = mix_remote_work(state); + } + std::hint::black_box(state); + Ok(()) +} + +#[allow(dead_code)] pub fn call_remote_reducer( _ctx: &ReducerContext, database_ident: Identity, @@ -55,13 +90,19 @@ where let args = bsatn::to_vec(args).map_err(|e| { format!("Failed to BSATN-serialize args for remote reducer {reducer_name} on database {database_ident}: {e}") })?; - let out = call_reducer_on_db(database_ident, reducer_name, &args).map_err(|e| match e { - RemoteCallError::Wounded(_) => into_reducer_error_message(e), - _ => format!("Failed to call remote reducer {reducer_name} on database {database_ident}: {e}"), - })?; + let out = call_reducer_on_db_2pc(database_ident, reducer_name, &args) + .map_err(|e| format!("Failed to call remote reducer {reducer_name} on database {database_ident}: {e}"))?; bsatn::from_slice(&out).map_err(|e| { format!( "Failed to BSATN-deserialize result from remote reducer {reducer_name} on database {database_ident}: {e}" ) }) } + +fn mix_remote_work(mut state: u64) -> u64 { + state ^= state >> 33; + state = state.wrapping_mul(0xFF51_AFD7_ED55_8CCD); + state ^= state >> 33; + state = state.wrapping_mul(0xC4CE_B9FE_1A85_EC53); + state ^ (state >> 33) +} diff --git a/tools/tpcc-dashboard/.editorconfig b/tools/tpcc-dashboard/.editorconfig index e5c16dc00d9..2686256015d 100644 --- a/tools/tpcc-dashboard/.editorconfig +++ b/tools/tpcc-dashboard/.editorconfig @@ -18,6 +18,10 @@ charset = utf-8 trim_trailing_whitespace = true max_line_length = 120 +[*.{html,css}] +indent_style = space +indent_size = 2 + [*.json] indent_style = space indent_size = 2 diff --git a/tools/tpcc-dashboard/index.html b/tools/tpcc-dashboard/index.html index bda8cbb76f6..4090fec48a9 100644 --- a/tools/tpcc-dashboard/index.html +++ b/tools/tpcc-dashboard/index.html @@ -4,6 +4,29 @@ tpcc-dashboard + + + + +
diff --git a/tools/tpcc-dashboard/package-lock.json b/tools/tpcc-dashboard/package-lock.json index 0f85fb53c44..8c8a636c4ce 100644 --- a/tools/tpcc-dashboard/package-lock.json +++ b/tools/tpcc-dashboard/package-lock.json @@ -8,8 +8,10 @@ "name": "tpcc-dashboard", "version": "0.0.0", "dependencies": { + "@reduxjs/toolkit": "^2.11.2", "react": "^19.2.4", "react-dom": "^19.2.4", + "react-redux": "^9.2.0", "recharts": "^3.8.1", "spacetimedb": "^2.1.0" }, diff --git a/tools/tpcc-dashboard/package.json b/tools/tpcc-dashboard/package.json index 072598ebecc..9d1a3b7177a 100644 --- a/tools/tpcc-dashboard/package.json +++ b/tools/tpcc-dashboard/package.json @@ -10,8 +10,10 @@ "preview": "vite preview" }, "dependencies": { + "@reduxjs/toolkit": "^2.11.2", "react": "^19.2.4", "react-dom": "^19.2.4", + "react-redux": "^9.2.0", "recharts": "^3.8.1", "spacetimedb": "^2.1.0" }, diff --git a/tools/tpcc-dashboard/public/ibm-plex-mono-latin-600-normal.woff2 b/tools/tpcc-dashboard/public/ibm-plex-mono-latin-600-normal.woff2 new file mode 100644 index 0000000000000000000000000000000000000000..67aeeb0104a02cf060b813abf6dcb6fdb97b7512 GIT binary patch literal 15620 zcmV+fJ^R9UPew8T0RR9106hc%5&!@I0Im=K06d=n0RR9100000000000000000000 z0000QOdE!99EMf~U>27;0EAcwL2CI^XW z41-u3>zhZ!+ozP^ZdDa+AZ)3Eacl~39)zL>chbl}9UYt(WdHyF=Oi662ICHZUo%q^ zLQ*>;GbAaM8ItT-5-Qt+L9*tITzeeLWl>r}wF^y-Kf_K_FWA<=IgGZz^E)Q+G}Cw) z)=R7f^%1Z2-V5frIC!c&4+Y$+Dm2XS%+s+B1Kp z_iX)Wm($&7IfsK3sJhEzfAaT9md@JvfIMkCJpv3F2ImHzNQ8FNxu!tpv9Wa~QHqf~ z|KB~o4LDhA1vqfP5>BqbtDi$+N3s!|0{9Prfq8}+PfhRv4cm6+w+e+;qV31V&U?_t zQnXRjVXVSE!s(#0$+r14DHc^D-y$!;7?cO=XV&1<8Pd$aCWN@Uved%-uenLHQV5yx zIvv_=SZb6y_phfqwSPDNOSLlP`=3pYpMFO5~X;J>&S?|jSNOitPywKOqw?o;7L9~pa|dqUHPSmlX zYWIOgY+-te;jOz1#kuU;4MNXBJT0*t&w(7yDz@P`4zb?OHV^E|ksPS}jBoCrvWkN^_|qH03|K!}y>0_RoE9Sa%47hW z0MdeDpl>7N;`YckNnBJ5zUZEm^c+J`{d;DiZu0o_{Lh{IDtR8}II`c-$93?B~Kpx+*~lN>G9ll%NC% zTZk5cuN);>j5svV(8iiW!sQ!V0RfsIs#U)ef#M_#cYz|o*CG&)G_g(2K1b{vBc|6_(m>i(+{~rM~2CXjxz5qSqL5d|Ctk~L`EC(q&Y&clPF*{pM*l~_M z4qRW3_-yqHMbV-f*07G$5;1qy=`8g`&2!9x++blE-DFW}*6kMYI7ct7`evJ@ zp`8xPJXf++$+f=J-B^>0VaNk%ZM?-6Jp$yl-UtLi3O=K z5Ul~GgD@w`j>f<_CU{XKKa^=p%Y^#f#RPTH8_2UrekL~I7?wyB3hb8-YiaUZ`=fP;!(1! zvvqaaz-8ax&gJ7qUVom&5A6%?hX7M(Ww1H22o%Yh3Y#?sx8+QZ#8VD*k314nxvdv? z5Y|z|B=#&RiThvVjz3vgYl0r}75#QP+DVVE{CP`aVE? z3xKK>_`mg||Kt9L1N{33{(YrE0DL^Ks3`!bi0qO#uYh3S0zk z12blUtWj99>MZJ58ZA%uA|9i!1{=Ya}jN=ro~IfKCUxP{0)f zu6zO9ELgIj2l3(4XA#%!y@kf*dPtqP3}WP^i~f4RCF6WrL+Tlvu~ZMll3Bn-6aIQ2 zmyD;MAw6~$mAh5&ef&UHsE;@D^&0IN%0sY4Kf1mO7JmkccN<>Km+Z>VB$hbNS_xNF zMEmQ3@)46FeJxy48l7V{vPSstfWu_TKDCLY@^E=!`I$wjh%<@$3P-t|Pdjtdl`uC| zc}2wuJsl;>f%6;e;zg!LdXbSu9>_6E5={hDyzXNmZt^pWY?R}GTS-1DJ>2XR7a?|{ zSr6VCCp~uPanb5xvIwGM0v~XO3df0->aPd!l6}SHX{iF*^W_hSB*L&@QjN?383mLT zAPd336u{^&fX0WQyaU+$ABaIc4ktM&g_u|y9bq*S^}{&iq{(5JDzRCO%%`y<1MZ*!jQO65Z$47mRG%SwJ2?PNo;4d zlWPsi7Hzd%{ECz|G7eJR#6jt#*GeZ^NUg;*iFB%~fAw{g2oo5=X&}VJ5lXC-inQ%3 zW28uo7E7LsAW*_+t=e-**R6Isty{g7N16hf=mExT-J(*|NSbnfd8t~I&Ksn+Q4l4S zS*6Rl-$-s5xZ{KL<=j_g(!e7e`$EZQjNVxfi1rPR)Jl4?VDuFiz;g(pf` z9w@D4y_R&GXeZY?(Q_`(%e^+pB1LJXqN0aNuF)&y%t+?h-@T`DT~hh%g58WsgqxPG z113kWM4oIzAlaWU2;>;+HF&#@@X}$!UilNbK$_L#rx9y>Y?uiH4yQA4Ss0Q*j6L3P z002HekC4csqxGcmA->8YgRFgC62z1*fswL5Q(#i-LG!1~ZqMJ-H$qmK8Bp(0vw(({ z#2Zgo351y7EU!-9<2^p*yF}fQ%Gj(Err+4l&lSg zYh?TTnO(%%VPb*PuDKhAVr^CF$dMQ}W(U1BS{vBXHb$(%tC34R+r`8f-iP}&*0C&i z+6w6wf&YZ9-7uFFKDRh0&L7>w2ElEQ(+B*6&N$F2Vk)k9zz#PRGj~#t2noTf1Dby>4I&4Qd1>eT<{%PB0rExxLRAD!uVX9ZId_j!LTQ zmV{ky--(*LoWpmf%C~n7k4!Mv=SyDy%uw1#YnO-iM&EX}IzDEgZ$sws(~hyZltzo5 zA$}VSTvk%bwf1>|gdUGW{X@o>^XwR_N`W$9K){kEPFhlkxwsg|5GB3}Df4H1BQAh* z4ko|D!85ggF1e^=aMTUr3OJC2BiL3OuSy6p$C161d44F^Qw39mf~fEDs>_p5x$Y9- znd8x^N>*s#M-ssU$^K2)rVrjrPdWq*j0-z0V(}+wT3`v?l-)4(PF`kfo*qKMUN&)+ z4zQ}U@@TG{A7z_!U!ZNsuvt@|J|vqZ1w4~49q&`;IeRW(JF`>hPiD!vpmNWD|5@dJ zimOxVEMCl^xP6L9%MSAcGcOzFmw@{ueRj<$=+%era**sYg^lOesoBxS$}g;ymO3y{ zAp0zES805TT)AXcet#;s*U;sQC-%Xq#Zbr&u~!(O*Tcpe|)oXL`3y3#N96udj>igr%DyLQ6f z>gP&&=GxJd-jU-|ox#;C5hZw<`NdSi^Mnj@OjD{nC=EERi9zwSl|>v!a)~oIwMTVRqUgEfV6`+Gj4_d3vKrg6nl;*jVI*O9gq+?d^=Q#Oc@)f6ol$m7 zv&M-jX73eEJQ^K;6``#BVW zePrl&W4OfkDT08;`*?SEFYk#aGNnx?pOJ_#7(pwl`?lb^u+jtEm>FSfr1leiF2_A6 z-fYKQ(mCNtk#^%9eThV_88LfuY!ldFXqPurz6i_elkd^8>~+XX%{@}Gd5frLOJ&~% zjc$Ry&oV`|0k$(qfu#0VypxlW{#Kt862!cijuEoD96c|;#-2{Z&xadf8LmHe=IzxJ z@3lv6z$CZSYfN2-bF&F|DVIXUj;UamBSEauVF^)&{DSRACU>YLK@_Fg-2~*`W2GG9 z8Cw%Ez}a!u=<5R3A`Y_xzr#Ck%i0s}ol408V#WpKb?)54NTXnj&^M@4%T=Gx0dHa^ zj8CA2oc!NF-tlycy{Mi9zX0G0wWHT)&)nN6+$sL96SVI|Wf8C>?WvQV<4HHyUiY<= zz5W*<*qZ~UyXnRgM#4y41FO+etI+2+%ln(tCcbQlpY`6HbN0p=@lu0GA6G_M(-XaF zrF=WPUPl=YyYt#+i8PZRR$O2bg4={$%Qf5`6kejrFG%4AtJc#A7XSNZ%Q7IJ6e2Dq z{nB%Vc>8pFABojGaYQO+(*5^eyoXUweV1>avyA@r1xZe(?wg}vXR^a|Lf1&RTw16Z zOQ~53$o06!GDgG`157w{N`qv`#D_$ILtN{u(yQ#=W-tb?WwVH&1%sR(SV#1>X<6?p zIT1fLA*uuFFO<{oW!INxO;2#=;io)jf|~4&qmJ2~>VNpdDBj-v%P5)`Mq%TBow>83 z+J4d%qrIIojV>SiRcNec<7K7XMC)~#m1On2z&)U5JYc`(N&JMc?ZmOEOvv%QOB`Ym zJA3vpT$PyfxVmW@d7vY#4!TBQV;LG*2_z9y-l$i(84F`@(slI25$Keq@ z6T=6k(kj)ia|X&#s(;06WNss-cB#WTZNd|JcT0~m_%M@ZcN@64EqA$S1Fy>(H5`+9 zO=rKymqVsQ6HwK;OiQn@Eddv+sCIGO3DEjmyYL1$e(5A$^GK^Vq@^74+2Lm6nq3cc zrQDmqA)*fItW+PWFeL>Tc1cr#^*m3RY;U%lpUV>O+#qpc!s*~s?kD=94d!9FNtT&w z{17a+RuvxSoXr>qdoKHN&!w-juloOg*F%q6x_d+n^yZaL{JLLg;L1x~@Lrg@{L~au zSE!meb%nnxRQAYqS-Kp>1PVWNZFzT(m(rAEu!B42Y|M)CRfWk=dL3Hdv+VJT%ywdX z?esXkl5#g=#G0^0z>v+#+)c^j?iA1NYb@DqKIS*bviKw0_0N$$dbgm<4AZ@Ezj|2R-JB+Z*49$ zzO*ZE6#Sv<%IlCX6xAdf^IeZL zf4vrh{%F;PQlP^ulGDu?o)!_-uZS%zk<0h)0G7oQ;?a_v=&c!vl{4dRG)FAg$qHq1 zU61~=xXhmD2^bcI?qd78q@G($|JrXqQ5UPgvZ(~M0T)y%3D{6h*!ctnbz1_0`(Az$ zxkgefmTP6q`k7VY&2@TRV=6|r=+&ho0xPEz)CO!oslo+ShEjq=T#~QUVXLInXsS3( zuVU+o&1UmkQAz1#p++YLew`$a)$~|HyQN3MM~cuh`C7us@YMxZHJhA*|F^0H?qvmQ zjTISd7b0nS?}}dFc_LvO;0dolLPKfSIAI+At{hYZUs2taSxh$b3dwYP9w#ArNd3LS(W4Ci+r3Qw2_2~ktxnQIi!_v-;SaDftz+tpNG z#puh0e&^a1^~u|EVN)HlCB(~`HH8=rEURw8o}?`bOAC@3kxHv^X=a_MoJO5~R=n(n z!r=3V?Zf#uMrN{r*h|M6ic90+8LO{oZ*3<@JTEHM=maN6!XvE|uy+BUZq@x#pIlo(Wi+#hm|{Su4@ymG0j z3h=$zVWuOEIxB*e@>!**j*I2gi9$~c*#ZI!%OVIo?=>pG{k<1Y9Pho<&vzi!5%fno zv60-9El9mIQaZ0x8qp(LPUc2Bu}28{bq<8iJoaM6l9Az~OU|x%aj(v-uPB{gqOUMJ zzqEPA#~OsAv@MfQ9{GA5FD4~Hsw;!wqE!k3m#YP*(htGCjT*`MKdh+!{o!D^F z$wH(?6e(?Cq*$N)?JkyR`I7Fc>Bf$^>uHmIOUL_glZqy{O^(2>Uv~vAwP1TQ_7@9n zJ#cArdBK)odSO@Y$}c(o(~|hN4Jru7T=np_3TdLNZNu%sDZ&?<2ZJF=a|jv)=e2-| zq-C`uN#bqOwT|zso&oG;Y-sD8OVi13+Q;{ermLu-f%QSQx(v0n*o#^QSQ^l;=)QdH z7_Dx36#wUS&dlg66-JCk;TOY(7Im)iq}o_pD5Ty;?Xu*Km|!YU$(jml^3EDvZdF;8 zq-a*~!Zt5tn}mFF?mf>&Jq6S}>g%l7ZmiR!4(6>}lIDzt{ax}?EqHM0VnwUN-B3#9 z%ir**f3C2At25kDOQRyda@FL0l>69ui64Y>@=?0*w|mST-SoUF?6T0wsLvVx6kdMDu79y z@s;TOIm>rwKYQL5jGNEL`Lyb4yoyr**?8smO6$lC$#dserCI_K{U@bY&y1$!uH5Hi zf)&$<8Y3>GvN4vJn@W6>XTJ%!%pP{>`8gZUHDGXbTV5U7$ra#zT2&=ZS}XrBnv?z* z6ZKE?r|+ZbL%%!zh_`(6P3w;)6Ai4Kj?xIQ7P_Ip^D(GU?NpG_+N-`?iNu$C6|E&J z)XqZC#|0h(9c$4Ds!)os>{XzY{Wv!Eu}aE%6&)v9Pfw$55Q)~&(&!sIDI?8Ecs^Ft zJd~#Kmj8$gIVQSWw&nqtm5$DanJ98e@aI2TBZYX+KP{J%Kl1UPy``*E2^hC+n8ft? ztY=LckI0^gjiM*dK3roe%FrivZ(HdO0DE0BR4DJwS8LDwQHlj{yGO4zH@lm;`Uq-! zeH(U%y0Vh2RFx5A8e}QIy;=_3_lg*!tKA1WvohlAK_I-;YSWX(7eT_d9%)tuU2DM$ zn^mKzO!PW*CJK`(OO&O?*Zs6exCkoyGxl)|`d7>!OO@$oZdx+`3PxEj@~3{G5dcAw zgIQTy^^<;!~)#KLcC=$rk=dxbUGKOGadi&4zaW(Y;cQU85z0!7uR#qJL3 z#L@OCgy>ne);P1r7f)fusSReozW5&df9kZH_tIe8eiRbb&rOUuKNN)ije#M5F8u!L zIM+jpk>@aG20c%Y)1U>euI=`jrF?lBqms%R9FEb}NE!;@gnWn5%AbF3(JS6+29|tJ zo9G@DS}@DRZMO0~@q2EsqBg%>RjryKz{-6uzS~w&hAnIM-WnyU*qw5Zt$evRR`-hh z_a5H!QwD#Nzcvw!{c<0j3qSmiXkgZ4Mr^y!?`xN5Op*hle-6XBbf5f;|Du>Q|CfZ1 zFg_cWB`(`Q#e78lGMgm6=r?8j=GKtCAF{ExjqKT*sg+;GDv!`Achux9cn7x0xhk zzXQhR2J<#)>?{5xvLkrZ!|yGSW2cONLm5J2ZEAnMo=X0q-wCaeVNm1P`gK|y>QM@T zaqgqpAE4(~m}d99X$>w!%xGPIzZqQEmx>||hsevgiL5+l^lQu4Q9dnG)xdChne*)h z%q|8mtGJ`9z@B5!J`X7)D7E+!o}4#BfK@MfbB9`5m>wrJplgPvs{92MHazw>i zpFMto#eX}B2Mk{bY=b5tzSCTI3My%BNl3N>nxssJOJngS-3x+@_?ra;Z(XE1xh@b< z1IszTIb2Uo;46JKnZ8<^x2`3S77aL>37u1?z2KQk$Xma<^7fe_Meuar=}_yDrv7bB zplwY{k2JN0l)q@+pW6=>=hzBd{Fv(4>~b_IFwT6QzC(!<+2xdHEO1_lq4Go zow~*g?^+)g;P`G4E#cU25I5oTz!*IBmC_1x05XqYVp9ewY|~H5^f%#TQ|Y)k|F?>_ zKL|!`z{528&+SwTuI%6OBXstvw3N|XfmCf(TK&rAH-VQ%>&r0*=f6k?r5qMhTJbTB zp$sdQSLUmAxN3z2Stiifa7?M4-xZuGFW*`r{k()vcd!rNetzqlVqy1oky@jYsb-z1 z3|?LnddHiusae%XY9UwK_~x@EvOaUK(q`lp66zXKRS=}6|9{$l3@aGwHU8O@HQUFQ zvFDr_O^^&q2e0lGSw=d`J70{@z@8wt(8E&K-GMK3 zzg!=nuuWa!vF^(WbW1dIqUrjy0HY1#^e(Ga80y-*nyyS7@JjcFacGa7* z6Q^T^zqdhWWiElh-tE!mx>Ud2b%|9=sM(*Mc2r(GqMD&5g+yPy-g z$vc<V?;941@l$`%2(D?Lc)pf zmK~J9ktcEhIhUc3YZ-cA%GFQQM?f*a5R<^1=J8Fs7J5i+Fw60k{1*#a^o%0=ZdXei zkv_!cTQp;cft+M$9}>PDnw+!KSH7S@tBcRbE#P7mEu=CEsklvFBn$~qq@U&TpGhde zP{J3RaMW6h$_cxHEKueE4D?J;)&O*E?%^!Z>_fTDoXrMh9qPCi4WU;Bw*XIskr6@1 zWfcvui}Rx_L_=W&6hKuN0EGt-H=_zEXn=lmJ*8)SoNm*l4*GL zik1Fc_cXm;m;zcN%go&I_mo&myt2-=%@fpWs_aG??g{!=V37T|RGvnM&y$+Yfz-l}!7zM5?-wne-Incw z++BzNURnmYyBa5cpqhK-?8=9>uK)=vGd;ZhZQ#5|zTf7Z(3&zmZn-?T&e8G1MT1Fb z|7o)`ZhM3Mpgnj`zol#ci!{YlgO>Xn*Bk7H>}u&Rc=|6VA^|ut#P%+I8VWx0E~nw9 zul%g`6<_w5>f>!Vyl2kyNSKaCf}{ZzLNGIExpl@49YwiK#L5JYhrcwLhjG#jLy9UI{z6o^DP3l0IEP2 zy@RD9t8~dX&?T0iY`)UW6AufecQ=|rYwUIWP;7@!i-yhDKUCp=!js`Q;eYur{W_p& z5dJ4TnbAIMY-zkYFi7FaYydQ#nTO#*suqJ3o@5L4CU#>~tFl3aCo>tSl7rc>gm{vi zUMC!cC1Eyf7B&YHL7xX1?+K)igfvQ`V>WCy?jAVo9(E78>o9N>?}|D0Q4(XzKFm%x zODsdI8t2?twYEx#LoIhJ)FX*SIkflJ;(c2)^}$q+c4QxVtRwr#qaO2kNA|HNI^s|E zsmDCwmmRs!zIcEA|Mx1YztyCAjWSl zc~kvO0vFP(+wq!xb)sGnuOR3t%^dmgdU2j0+Y@okFy4iT_3j{|2tS`3;>n;-GY zky%<;gKL$$tn#U@9*qdd1yybkCkUV{LY0g55<*VeFH_Z7c!{WT zDHekk%)PDH-|&dk@U%5l{{oWc@?ub&cJ2wO;d~7tKW~Q`&es4b>5y!?XKA5AYB-^U z5Dy0-H9SCDB2W&a(^l7{>l@@Z6Tys|rZUzv4CQkun;$XRp^UzLYHSFtwzkJcXHYvY z&w3rJ^mn+{QdjOZaQ~+)#W`=!7jyy7{|2vr{SAf>0nFzIbby_d0_Xrc8(s=}D{J)i zj53`mWsaC;jK}YHV#|het1y|beJc7C z+Mo6Y^Xglbzo$4!qC8$jzpYHEhGZU{Aymh7-iCTu9K#xR@)M9PNE^$SBa z)ATjkj#U!ZFO*fIvCgG(?9?#+KUPqcn;C(z71;(2R9g zp6nw&{LIaMvg=B$@0vxEzh@|w0Z8&Xr8dZVNlBJ?!2oK>BP6us)RtF>Kmh249j!@x z;Y<14z5adot+4k_VxEfe!Xl9f-QZ0+xv*W(!9Jp(ef^DnV=CI@H-fEwZ&bEF`T4)C z=UaH(aWDJJx13E2=|03nQ%j@=3C|RVV-h5eDHfynR6lTTEIVzd>B)==JQgaGuU=A` zrN&<)SOV~SMuuHd$nyZ>sm^T#c7TlJKsP`)vzqujGZ;Xvw1>dxOew}afC$1sR@fpZ z?2ub?wZ(0vA74nXwIH|ic6w{fGYHvWCmFU~Dm^?sFp!?5b1!*AWw@lU8s@C@K{A$h zBDt<(a(_?kyO)OpdpwdJ(t4q0e(`456CSz(CBpj10E4`@} z7J>kPNzsBuvDsmg3KvLom9DW~PbBUM)9aM_$A%5dJVvzO)>;S=51fymX$tMcnR^*9 zf!M!jF!QB&ABM<62~XWnL1uq4sy`5G#r}O=hEFgZ{2E##dF+(|g^9U;5ALXv)E1V4 zCY|VY10ms=*vjZNVabA{7-*b20kP=zFjEBUuMPepuLK>E6)sW2$G8i}rDkTLpOUD~S7+L$&y}_8223ado6GSrAr6~F54Z}P z1MN1sGmIxdpk_vvDOiY!ce$hUtt?2na_O3EGGzDEs=A?NbqVFt`B_QXXDoni+JkKC zKBj7x@vSNAFGQb}fit;kCd03sc)C*@M%W&!>cO#q69w%-hGg2oL~7Kj^=(eGikt9GV}dEL95aa)VFAmD zV6IP)Sn#=Ov?pBNmKK`gLMSk)6f1l7C8`%`T`{CnqGssU2VXWmRQIRXYSlLClQ_k$ ze%G_0p{W6xnNPPOzVlphv(*?M*yAX5FBh_+XYfT1b!%&z?_zBTt$5$82uM))vS1USxh1 zJ{lE}x!v#ub?oN--ey@WO{>>tvcXa#ZqN%XKhOt_2wdfHARHJVE`k5c+(Q?s1keyy6+3p%fOwX!b%vKaPrF&(L+I7B!> z$JiR0+mXwa(<^qcO>T6q3fEVOfPGO4_U*cI{W#d@9lJ|zcp%-mhs4eG-N%Zg_d7jw zbYexi{|IS&Ex`kQBsI%c?6wDzK>n%NeauSG@)?LL!Efam00{lQOI%-B0l?axt?IggQJ?iKas_ow0PyDi z_fj8oR$B&Dp$c4d&h}zm*^^umjj?NB1C@-)3-H+G*W<6{=GiLb^d zKJC8yz%*+)xZ0YwB}C8bW?;>VWN@gb^4H5j>Co%>hj-G;B@a;O#Ff8nKQKYU86N|_ zUue!f2rs?qEf-mCT199PDSzr5m*qttc zjmsPB>}9SkFjcqk(@1AKdD3(*y%D(+Z3u0vy#wq;<(uHvjWurhEI4xO>{RQ;Ne}pq zk-TmZVKOSpl9Ij)BmV5KLH5LO3rvDj063s;O0Xc2&`6%y%=OGxuGhLk8D%lduGEHB)PMniNc7z1&3HUk20& z0#<2K1_dk7v5#r~=>d-CL)3`u5HjXR<)2hx;G&z|@O>AK#6zFzq=L49yhM*vB)H}vO4O*IA7$KRQ?rOq-9u?s_}3|U7-(0S zB7|_r{V9uz(Vnz0I|bw>bkM=^aPzEXUItudnehZ9@QyKCPA`d4+&HYTdFaSJ@6VfE z=mTF=C`y@pz2FAS0$8KzCKJNgEk;-b8=reQbP)GLvh#MQbB291-5eQ*6=l{A_c}<- zBdAna-vYuG=EMzmoGGO|7?ax2c-n{J=u~vLM-S> z#33<_DA$F{?h9&b^D6am(8vT^i^PCXdILU#;cC7dy=_Z2|DW}-OOgu{j?jQi17mL3X2PF#XN!00_ zJ%h{8c7pV3<8Z3$Lf(19ZaA&ey zYb1)9__i2pM}(pxt`3Y-!sEgP1ziB+*b91-YYkOPl|*W` zkRqMuY>A_qt4b*PWOm_U6aZ#SZMnvFMTIN-vrX@8_RQkXN%RwBJ{NULR3f}V2&)A) zF^$N!_$nCD_4OtZQaFVcsGD;hGuDVZUZ}*qb%8!`_s& z(5DjNJEm(>(4AGoKS5gXlP%=_?h^JEPVhCeRi?s4alZU=CG4%g;11u~w(Ly>s1lsI zZ1Mf!Jny4&$A8EuBgfre%a*@GgV@0b006BjJ(!$7jA-0Om(52ji;oaFQTrWz6bEbi_Vp$jrHcIo&S30c-{n8VzUn*;rp4d}!9*vVCgnID18E4B zKLmOJiEyvv_^@5AM(QL9>}Rx(j`B`k`;ebXLM^CKD+p+7YyC(Ln1wd_d+kZ}>)hIArehP6sx?w0ARrzyGj~8itRL?e|!H20s_U|@e_B}SEa{(1BuCR0C+z~8UgtD2|4z^eEYYkb0*M+00`I@ zk3Hq|PUU^0l!}vNw3J&`41Jv#gCSi-L_@$Ppp2b4f-bg83{Hr`h$s!NJV@ojWyv!_ zq2q|77mBSbKe-8bxs)blnU@Kf>P7Ps3X@9I7Pzf@Y_^LQ zc+xI4IyK0&d(i>9(q{g9w&_$SP=R#0vLwqhMKhgQ^0aBy#$Pd=77dcMX{J?1r_h$Zwp}dMF?4CKEwlojOFph{fcD(>Ipp6wu)PRKlP*K1EZK78%9F3a6orZu zD^aS9TDc09s#GJWkgqowO=gSLW_LIN5P}gD+pnI~xKn>qHYppjA1D3a&4D8)Vs0#o zgmcbIX6ZO*=3FpL47=oli|)GTx*LuqG@KZlHh7lATr%IR#;_?KK=IC zYiXGv4Q6ObG$eku*)Q~XuH^1m>40mNl_@M|YO4lKnl)}i( zHLG>M4;;=M03jGbF`OVNnqfI!5GA=$f>8z5Fn|AU6?}IzHK;%=;`zY{(iMPF4?p%n z-&}b~6~g?b5AN4=i~s#mv!6{L6E!#K%6@yNVwnRAi?goyH;0PcTiTgAWG-zNhp4GaWvE-+GHU=WuA$pr=iBje30AjLuvMHN$A2_;ojKqZw` zQMC$dGQ-<+Z%W|2f&o@U2?Z6X%11@`6ROu8?~oHTH9g%rRz36xBj()+FEonvAc^R9CkY_W7IeB7T5dfUtotY}Kuj96{o^s+6! z>3#!i|3F-LA7N3$06+U$uM|-L`2KrVCJLeQJ@Q}V%89i~P-6qT$MT4IE#l0+iSo`$ z{le#7HFtt7%X{w56bm`FY}XEVi;|C%GQC=Aoxmx%*DZ{jBPbl(Fv{!`8gX1ibMUpK z-~Y;2c4jAEOZnKVPV}M^>&9Pnvh1toABKZ%;1DIu+<^#svD&FS^~;Gz6MoxJrr1eb eR3cDM)T4T~^dNioH}c94d6i%Fz)#w|`vL&JyAfCb literal 0 HcmV?d00001 diff --git a/tools/tpcc-dashboard/public/inter-latin-wght-normal.woff2 b/tools/tpcc-dashboard/public/inter-latin-wght-normal.woff2 new file mode 100644 index 0000000000000000000000000000000000000000..d15208de03cd1ad7c5199f0a0ce915fe841e4722 GIT binary patch literal 48256 zcmY(qQ;;}L&?GpvJ@Xsewr$(CZQHhO+qP}nHuwMC-frAwKU8*AMD|NXKXg{R$%!%o z00R7nZhrvO{{rAX{(l_wf9L)u|NjS8sP4bOMq%s~dp>bh0mUc_KAHc5E|@-_lCpqu zfGim)KnN?HH6jF5Ko@^N0Sa_D8~}Y_HZLG65HWB(1_(ZAC@pva2n@KE(@cV3_rJN@ zn;~mU`I0RJ|H+Cb#jiShU^cg6`%XnS7bCbgg;C{t9iDca{lng-mFvI%FegiD6m8pN z`_JD$G)3d@FJGZEOKh?vwFPSdW*2cc0wYxVV2D{tb@kF9Yg_0c?AmFTV}z(AIX2o% zbRV=fRtM}F;{$;rLcW}5?IlE|+b1%2-cvuD&UoawY`)@_X=c6RC(2%7q(kw8_;ocFmVabMzd#!`S#lF4}TNR1HVRdk&j$3c@!VLD?fRP8wTo&;#< z^>TZjk^$};QGy^-M1Xk%!mBnDHKrXwS7bP`v?pB3BdhmGchcQ{0boVeg;WVyqG2f; zY|1^n9u#lk&o#A9`t9rP_ixW`DD#QLQcmKQ$YKdgI~OjhE_{n8(pf-)GRUueoVcnv z$G7Tj(K8P3Wn(zmbquoXvEiRH@(mi_3jsYl+P+XFu2Di$qk6jMI=#UO{J$P(Ny9pC?Wz3Reo4FL?m`Cxd8oPci-6PtOleSE|_=_gHmg_&wdmz@LO8Qxp6rie?9yykM=cb*R2=4msGM zyaX^JNU>pmu{KJT{s0D%X!JHlP>g`xuz7+3V4$%DFs0}pJkb+T*ciuGS}7PzI=uoP z8HKf(;a6Ffm?R~?mZZ#*a`_Tkg&|`)J}<{DZxhpr@z7D}Pf^LC6CWbgmTgy;U2QaS zN^P%7OkpBE0Wm@tzqtIz1KZW#F)@FZf*rU&;GXHPz1oYmXG}OUpekb0h&Q61WKx;lXIUZ3L(l7M?(DVvqz!wbpx|_Q z_+M`KSy`*&O5bOl(`6eJr!q%BTs`pld=W6a4RKW-0H^v)>Vhl=&i;x8+cOC72p!zh z+iOb?6w1Cn`4J|a3VZ)rv=O#--vXby8Ken84ax!F>TdS5|`#IG001Vf{0? zk)InKeD2+tZ%&ldUj_k_aEd6TFs1MBP^r*j_?_*tDN?3|lGo4eE?nN($6noD$FSU^ zX{uj#a1Wc40-i;@b`z^0!Q7>Lv0{b0d2tt_v9*~pIR?iDbPPRAqF>zL8~ zTj`5Jyn-Z!z2a~Uy(x`UMQ~96-`}29rrX=+Rk2!zwaHDk-zdR7 z88wy=gejX2Q4iV7(o%}`?B=Sw{lQjKQ!t>_^-My~7ohti)$qngzx$^>sB)R4!s)a6 zZztTV%_ir^-&*fVjthK(thnK>c{ZGrTf0@b*E84SUM4h;(oymAl zgx>`4AnjyeQmcO>z`YSh=)=83Nv9o9v;Q8_0_DwbF5j30SAn4^MQRJ~a$t)oDD?wh zTW8xG0qcOZxP|(#%K#cIjbuJ;Y1g7Q_aMbDFmf^GEI~3q5(?7TSDcf@ROJJ00 zHJz8%1Jb#HL#C{-^5i^p*Vs#2mr3%1!QL&0*;y*#2EL+jV6gzP({8B69SA)B4&FAI zfbg&{p;&xk=b0Nx)-y&{MbMn%pVvRI^MK-x?$;?^t)PsfM(;6PbdE{-SRshkb%nrS z`PeQ!D+?JG*G*gfl|4WOf4sl(*%$Q42Dv}3XQBQ#gj0PiaxE6+Bgp4zs`459=P86} ze86Z?G<_fx08(M#e$m4~tSPi)HtlupWcrxMdhX78!O(ib(|TlYgEEjIT8WUAiC~M3 z!b|)RmxsbjWKjVScIPn4B9tKTIrA#U265APjv+zu%)N&S+wGE}$XdW{Dp$A(&&MW7)+d@!BM5pJZGT z`J7|{nx(%k39o0?nP?;bihzskYV!%IKEm6FhS85$cGgS-)6Mip`Rpux!o9185wC~+ zxcM2;7%&{TQUgNeXANpyEV&B}kWt~zj?c^mm)v10Pfomt3BIv92+}}8`9Q;b7_q@6 z1uqMtaO@JI$WbCEl*mE`EOaA1?5HXb%32vi2?r!O$tCmCsX`6vC^Hq0R&i*>Y`TyW zE+G+T6s2hgM??$%bm^p|Me;;%^PVsz?!Fuk(+$eB;Z)0Nf|RL>JTwv&5rP!usYbJn zu|d`(8ue9ycfa7;{LvRT6@GN%#N^bJw(dydvf!xJt} zRPt1rH0pZaCo6a7RoS_TkPVoD_XOKT)%v~d>tRL-j>biSMWW*t#dkl&F<8{9YTr9U zLYP_lmq&nnU(ZXK?x=ZBgRX9nhRqWCk3Are?z@E5$dn?$1BLr5l;FRIbw)q?LW)M; zm3ZmUYdVcDUlW6QX&rHMu8~9g@oe6vLLOadrx{6jmWY*|Yv?)E>1-t@LrpSACCk0) zw!a#vin{a{-j9t$-QGUh?enz~DJgxcDl&A%pCV2tv{z^Hi-)zHi`T7%HK8%nWS2V@6^F`g=EWY|*W`q)TbBYbqt$IjXcRrZWI7BV$Rmi(;jywUr> zQANO4BMD1kQ-Y3Fm4O$wioxXv&E0rS915#9j~Rjl6w&!{XqH0Z(^!P8 z5Nr4VK{gF6>cV`R3UN^5JjSMk>ik=nnu6qP?fIahU|g%_U7PbsI~y!DmJ(5%l@d6s zv#DlPP{?-M+trwZ^7_cb?n(X3i?O}${IAJvdNkYIX_JWa3a=D4R6>Q)a2qo<8!>U4 zLD9B`ksaIDy4PTh2)4+H%L6ugw)$JIrD*;l6IXSFta7%Sxyv)Jw$4g^J73wM4QvsR zCrpLHtyGcFl0yeCmNuKc9=P+i-w{dPS(FGZsvX0#VjnS!O;$>|$L3lJC+(YIBaZ1H zQtsql1ETsIH8r?ulNV%-wQIb5ZpeTC^%oOLcgT8YuezbCGK-`UJ4^ji*()!f5U>_U zn(64p`NTsvM&U@J**&P%+IKXYWhxjOu9T%v(FIkSpq)``TAw2HuJ_!HAT~D+U$M>( zNZqb^-_5f+$@`$}aEmt@nWcN_MY$(?Tt>gbbG8RfHB_ zJG=^2B(wBivHgl;cSy}9G zny_v6g)cY<0ri@ZruGd&f}C}aWD-Pe=EC9YSfc>#sPVAX(#;14R9f4snq0SuSL;t$ z;ua5UYn}OO8rkcmh{Vz;7e7tj0ezaH_eTbr)UcP+QrOL0ZtoggZwxAw3!K}D6j+NR zQX=|SrosJb)0&aGq8OTa4PzRVhLjBKY@$usD9ysfM!4L%Sz993`Jit^702e2mk5jU z3akwQ{^i8W6STs#9YVRJP4E)`+Sm6#1k2#$PL>8@uIk#W-R8A5Z%jrL9&M?X-w!_y zKInGdHkZ{QYxAc%rts^NrKJeieGBZA`4siAdx`n)7IUsP=mAlgBp1I(nFi8GMtACn zDfTNnbed)Av}P+Un#WCrg7fFJ&IiDZ_S|$qj+#6Z;)L9# zfMhFywxd!h90NV-MyxUWmS5sDC;%CF&WsR~0h3R4k{e#B$k>&7v%7dCCf^2x>vRTDj#RiL6|_c3r0j6Q=w*zmcPNW0hU; zKHPmI63sT8Jms$S1Lz}fIH~QEJvt91hYmvA7450dg5Pl*o!jtEc@CUVrhhhe_4hGf zNdm|m5;=x5c3b}_>dU;Zf#1~NYJyGRN-u?}IU| z0#Xc+xJGn^xxqd}x#|PMw|4-u?HkJG!i`AT;cq7CE#8ZWPvlH;Pvt%jmd#;3#Cek_ zGuU}7VS1l=WG^$Gh=BVcp_-(WDW#Wmn7=cQsb%Co&_I1^c;5 zg|rnwhr=r85iq7*s%;)PH2zN++fhREv3>VmUA(vg2i*#o>k555--j|}d69O!uE9km zn>NR?X(|fv;=V|H7(Up5;PlhJjmFl1-5k&}7h0g5IYL3zMUcct$1aCajZj<9pAe-^ zO>M0p5AOv#ZiR7bkvQ(b;HNXx<5)+kz|`HaLY|6d#4^f>fXivR|E+%6NPchhq}B+M ztARuTCuAG=*5tboz2XpQ01ISlsv4=>k3%Vev`_MLfIDRU05Af}r<~0f@+Ti< zAp7;e?OSUCiDIL&X?-h1`dMZ6N=t{7I>8EN$qyIcgi&bY{XEp8AqMUZ8qdYa3DpjxSoIK{q2@i7>MCZ(!R100F=&DG{r5cyJTP zs8#ZS9$F0XhGX%#5{*pEgCIp(yF^l39O55+#qSL=$SQe{yMP}<8ZV*u52TQ{zTxjv z9<*SKZsv3S=oa@N!d*Mp)1ELyG?5{qOJJpiUm5C?URc&Yrfx zU11^;N^f*l;y62G>!c{B?*FcweD*9Kb+9*%=%U=w4N|-_a{=sU2Cwl`NxIsSzv6bo z?_RT0)zNP6&JJQUzg245a<1f$jgbd1!3q_hsjbkbL!0T7gYL$(;$;y#D+cS1!t-L=ClH~jGnT~jOurKzl zC*$n#w)?K+hNB{8VD6w35(Q;w3G}c#Q3HlzR2HoVXgIhUYl<`*rxN8&Gw(EK+$X9n<0WC${ae8Zq zS(j72{eF4n3;9(a(}G2e)}VQu%3dmXWrl54lI8t(YHzqT6e^ zn!5uKFg;d{9#WwCwW*qjSPh<8iIqA#$4qBfR@n}mUAxqVfXFb<{u$ATe>B|N^OZ}i(nxgYk`)kE5{!zhKq%o zm{8_;nyOvLIP&^C5|Lo;(3Th>%r6D=u@bG^z;;w#krph@W=|%|s-|=0T$h!%y0y`e zf7ocLy4Got3yw+nK+#8lPw)DIK=h0t;er7Ug!<^K2GTN6$tU1vghEOhzar4YEjX;WAvyhIdGoScwym$cK;+cT^3wgg0rWSE{eY1+6Ke8RRqd$PQvjpBq%MPld4fdGwS8j@gvL=uR_o!+R@c=+NmifqGa=>86>8N|p`W@K zp-F$sO;G1C2g712183y1?<}ys2U&{LKtATqj!uzmJFe%(Qii-Bk05-!J)a*Q27QT$ zI%Sf%*L{VF^yD3OB7iy>LYuA zyWPN1`+HTo-yCBcKiAsfo6`;_cYm`PI|`{!1`-?)6vqWSRtoTr%|-?=3q~Mm^WahBx#TOO@zRps-c)nK|?fI4cax4 zE?L1$j$m)}3;%UPXeF-QWal5z|83t3B(6xaJs}Oi_NMT(9}M`zen4xnmf4-jJK28~ z39Nv(ZS`CI+{4yrO7UEYLKZnPxV>aRCXaUp@uF;-zmZ(y{M}Kj${xAwEuQ*%$?eesc_w3jpQBQVGOZ?mC1`3E^+R_}=!iMx!re;u?O*>9=0%@^Jdk8^Kr$wl+o0^g*2zg=QzBUZLw7 z(P(L3t;5W*!B(R^tP}e~yzbli+L`6XxlgiD=e;J$vs>Y^(*Kde3*GiaYow(3vv2rf9#Oq((-z<#oD56=V z`WgXGEtNa&#eRkF_ob$(O_q3T$QH9Sx~jap&D8Q#E3zD>D{VV@;2H1rHmWpNW}F1k zJWIox*;lfNtG5z%#kw6LZCB=T*;P?fKXYYx<+#h_0!FpZwru6F(_oP_(5wfPYZwQ; z@Urj9O{ZzUQUSi28`gV9SUTLj$-7fq1FdVO%X3<0>vUQ*9mA7Rv#d+|d#iJqRwYAc zeFIgl>Md<1uU1gb1WdI*l)8!uYGn%8By9QXJlHKnm*F|YXp_%mI&#G^8eH6m+Z_vA)PtARZs(s%7~n~q-nazMLj zyTSuh%S}&uErENEB2qhPbX^Ys+$0its=Q+=;F$F zTgO$e)4HM>t9bLjT%cPdc8xcNxeP5UVe;b~nsDX2=m|c;?wh+8@hs{4K-j(D3oGN0 zF?o;O6RYBp@mJ-&jzqB)h)W&Vyj5T7Hd+Oov5PP=BUovrO_o;#3Q^%P;cW%@*OMSx z3*PKU5h|F$?c`>|w5+z?i(=leby-~1Ivo-`Q@@0>p1wW=JPuhL@qr z>AF1ylfky}8GUZQ?f(VwgzL2Fvaw_s+py(U6L7=pb34*kU^rl;(Hpq)Z;TzSP$Yn) z)whG+47Q$x4)JRiHQ6ZJSRk#GQeurRfpykFT?@&1y@>*&ERAl!tj<((0GHcxUhazH zz$|2gJ2&iNEjb?8Wf~ zZJ>EWm%*QQ_Xh!NRReSf5AIJqmx^yBU>P(9_2fXTD{+Xr*Z`{B+gA-5-u@_P`fuT_ z0KauCG5r-wmdf-AD;-B9cQG+u^vIm>#Uw@0sN=y^RZ#O<B^iL=W2wjRcLF-^A-;|^rb{y zqL4{n4JYqZOVfc{@I5$E@Q+Kn*TOPoD_l4<^f?=00@xAR|`M3Q!8PcWS6hKokKx#Jbn3xeK z!}j3yciU@jSD?YX&t*=lMp7axdH%Jvqak~0ds|m(@!@^mDR7xFXVX1c9%=(VF*-#m@Sd%t{X|Si}y^@gk6@PkZvOf_v^pnbHtvklw=uZ5+Hk zXu=xOu2@q+#VFSI4gM4LhsFK+gZV%t1m#nThZVXg5A*NRX6WI@zBxHkuLH1QDZb1R zCuc;jLAOD$LD`777J(ylOK5|*boJ5|$w$2tDs2v{T94q}yP4uGV20Ntt1E>x8|DhL z!(8e@-HVyZA;i$$>)q&L=r7?JJl?<}v;!kMOQgvH4#7p%tzCf>z3r483oP4#$3AS| z_F7zhc@IFhjkkiAW$u{F7Ik#oLG0izb3jb!0Y5XHL|eOw+1)2pbH5FR*4b{t4~x~l z!`*M;!MXOoW=T9r$L4AH8#iL(g6`JNaEKS1y^!5M0ol5lk|2^}es(_i&V__)p4!B0 zJqe`wi2wSNgROu7rVcg6A0_u5O6M1CI!v+;Tz4vRcNf40kq;EaPdZ_MmJ-ZGE1IAV zNY#vNmBuedm>xGQc&0%QFe{P??UI3!nmNSS;+N15BQ~sIbTHn4HsJ~I}IVkD?K;7ERdEdDCq9$sjflWL<6y)l3nTR@czdNCJ_G2a%Wuk-%7bb!?i8+juRW z9DS+gNq3^-S--;OI(C|BLwJ5_hPvklyY;URKo5}Tp+roqkCeQxGx$W%v zfObE+c#}oH2QZ8?gJ;iKmFF%X;7GBdCZZyoApUD@bNokQ1wRu*<|#`%hK(fabH9Zft34Vq;}?w|{iZV>uO z>23nHM!!Vacg^HQVrFs4IWwh?m_S?=P?=FTcAhK%S&n7N#9n1Kufe{D%DqZ&5!%US z*gmqB&!v;z`=VMe??K&9tsLb66mXi?qiggM;TxL$;Qy9GrVKqp1;6j?jSZT(*BPd2 z&iOV&=5E%;e;8jJ++00jaOyW8ano#rydo%)t-47%0xDaq6M=OaTWSDE=>2`~F}?uP zzIppigJw*t*k&00o=yNiodahbaDA|*LGsih-n7iRNk)ZFA^)(H_y<fc0Uv?U;nUwAHnPqBN8FLI0(y`l`2Cda+-&R^HqVY7Ea zhLd(txXQtFwr$~YgA_cc_pImfWbsiU^WD$j{w=Bj)<;Nmhj5^v0e;|*MWc+jy;@*R!+9yS=*w;&$4|x%$~4h;Gi{yK@_*#;VMb!k!reN=g-olGk*V zq%k>ZC}9mH@=Ob@f>&lQOQHVYWLx>P%$M+C+inq1rmImrqxQH6HOi{KTO|UfXXR;U(-AW{z zB~ZYiE=g&LrOD|DCQ53Grph`CK~FmXbpu5dh>VJy7N4A$nvfF3U8E!*efPt#kXeX3 zbd>uRh~9H-Qs=&6)um)rR}glK;xSsBUtS!orVxX5q!XSAt{Av)J?PvQq;y{+)p}BKt;LQpWd$0r8Zyl- zuFfwJU7IK_cmd`qBwO-cx%)f{A*qTGk=m~&VoaozjiFQK(n-8B~#pzviqSp)v7R8$#+9{Rwr z7+HBq#nq{Au<3e?+E7%PU3i2>P?>asIZp9+yu9OGBBCObvZScj3(DO&q9U$ZAeYe? zR|br|#2SNl&m}8V)+M4q=X(;#WFdO9I^X0(!1na@yyY|WSuX7Z+T%N9^Igr!R2R+l zi0(WzW3ICM5bJ zi9$Gn#j^@(@Yux8JPdhRxJBjuhp?!Xcs4Ilw&!fpc1ckX5FViud_!A6EsulHIWymv zSKS6G5a!3Up@fVD+rH}8tYd{s$s$F&CR6+0-(1tvGPm{hcETW9!)36SJmdue*lnW2aN|y= z>JMIEGx5x*815ZUCElK&J~BaR7N8?Ua8x)w2lk6XIGDM(sBA!C@euv~c|){xwe>Xh zHMa7yIhLj@tu3xDuP>m0$KWGsIJq)0C26s`wgkN|$O}Mc>iGLdk%>C8?I9YX@g|l< zhD%fRv{mU$V^|7n!w$C0qmf*e)~!PAGpP=8c#h>)++2iM^Ta znU3JO&mHJ8)Czlr`b0_4E*l!900m3(I4UWBoWFxnFGC4x1(p)3v(68OU<0Fy?z=WWO&pP@hl)Dm3|vjccA<4a%zbiIY?Qc9(9Xi7Z( zwXM;r$8yW8TQ}rui#CVI$zpdmiCHWVgC_f&tg6C*@M0q=TSl;A&r+3nc;z^}N$}XV z{j=3khu#yEnpZJ+dzRB|U8?^Lb?F5*8OVrW!IIOO@_c?gj>uc-Jcse@gd2$Lu+Yrb zEfs0kXw!rYla{Uv9Yc;ImbSMh-Uxrf2ydTPD4OzFDn!R8*B*Kmnu<))AdV4vmk|5L zMN>gaRp3W4M6=KS3La1(Cjv3l3&19D7>VFPb*=YrZQ{W;f;misTFE$S>X;f#qpHRD z3D@C@iYID-HXISHCQ*0d2@<>r1K_}^2ZS_9=&+N86Fo5s+8F^oG5OOI7d>&X$(bnz z(%CCD`>r-p+XgQM9n($j^#ZH*S~p`ksB$&kIVUtsZYMKk2M*x2!1*W(}M6x48(F-gSaz z(X{)FhRTiSC)R^Ri~%_wJbIq#AZ^HrFe+yd8B#J!tH|Rlw?HXQ@1unOBw&N>lS)}6 zja&?!XwFo7qYL-NW%g+D2fs6rb7rkP-}L3Pf9p5gg8L4DTa4`#vnsSp2^>(IE1)jJ zcSp)6nmPZ*x@|d83#NPUX>*aOJw8T%r38txuHfwjQ2@Aj4S11wX_mQvd5#a_D0hE- z?at|`04oFF@cjyz`9!7ZSG*c2r;Yr5OK&)OV;CLaAc3baf2fDy{ry{Ff70lNith@o zO2H!(-O9s$$kQlE3A6M0M1jvS3{G83-V9>czHRu9_hi|;{h$)%614TVn-)(% zvV&9DkmA$K>8JyS8=C$b$71YA%vQYI@s6hOTINI)x}e7i1CEha%48aaN+J1yU`t*m z8$77D^}zg|vNGkmWUup)RQMQtwpZjhx-GYW_lHrldCJ*5dUHU~40M#*Vaj{%dyHYj z$W33^TftOrmxvutJ?h~^54h?ugNlC;q>u;}QtMcNod4f!%UE|$J3NmWZpI2?+q&5h zA<1X|d9ssCwRkyWBAgw4JM?4?aYiYwR&>8FWm8DDJp{^sxj5;!_5oh#kBKo)(_$j{ zfp&!x5Ef|*%+}JKC6k6KH+6s$AADD>#;@+d@!XL)wdwPwShGIAz__Of;Hb&(VT;=9 za4zqVs2sQ7WVw@b!87*P@*!#ZLdY%R6uA+WrDWv0oob48NyRB`SGVN{EH?*#4kb|aXi zAP_@nWRH34DNIS$284$ZG#NCe%%F{=QS+i$Y6`?qRKb17NV3v-0(NGvc?qXcn2lo> zIj{K~TbG)t>^o_)OuNhmr($pY)no=%jK4H|$#Cqt%-`zhDz&fWYD!cI+6duDBnk0> z<0KJHiI91|4AW!+PxFGzCu~#nE%iC5%JdmxsB!#UM3^O!Sjt1@0xyFz583M6!ytvr3|Pj4mBnwB^4D^L_`7!BmjimKXRST)+rA&O6t|`>1k;d z6%}FOU}6aX{)SXTCMKq)ZB(+%%F4>{a0rP+0RMkbwf}AYA3*(|5ovs<|HA+d{xAC< zg7ICf;hh(exGtfBX&~!~LV*WOLvgJ5az-j%{pQrI1}jP7fI=5^WMX#xC!+hy_uD7Z zSyfTi#gby80CIKewcaG7;ACTaou2l6U1G_hKJO zj>2=w{0*Op$fK!Df%*jP8{V8744`%IB`fYBt#UNr?MwJTAUp;b4K=xDdM|V zI?4zMuSH*~)+8ARa}l3{B!rI$^iMD#Owb)0=Yw6varoXy>?fqLG=>wBRI~R*s!FMJ zz9?1te63Qm!$8Neau_j!|2V&hke?_J9GbFtZhvoX;9wMD2+@FBT|d7cGU(go1y?6} zE4O#vwCah=SALY}tGQf-QqXZxI0FEQdiT9xfk0?nL6Z31NF*kko|avCA4H&CjAY2^ zjk#Q@O0mPRgRw-RmP2pXuE!Ulj*wZ`CA@N9QCWZd=v6~NA%Ng+ZNvwiZUkZf2-Pj* z9TJgHI2>Uh5EL4XIz8}zAehe|00as}O!gn-JoYvURq=Hl>7=9h1CEb{a&rm^XkVD6 zHd;C&2i2TN#SRRtswmjKdT-GwI)Gdqz^+?arro_;;qY-^E~7R3r*4#WPA<&8x9J`v z;%vp*b#{B)F$rrg6LCl<^I~18o!g`ug-R)o%7X-(x{SeVQpQr@-%F%i`-`1`82mSB zH++#tA3y<8-6UQ$?3~3MfbQ-RZ$jYtfMeKP+QkT0kP{tU7%7=Ni~MYB`}_zDRtO6X zFGom7mMawg%_}&ffVvi4GdJMiw5mMz`%RyYMu$Vxvzho|;=cldVZcG{Xj-h0mlPnG zpknIXa_39IU#k;p03kdBAj^XdFoPm);-{Ap4`Sh~x8Uo4L9Eh5*n11Nwh(7r5t(>F zE4uS4x;7KT9Pa!yg-8S|1+A$nJ7cUQs*woi7Z9LFu3ZOX4l$A zH#IOZ5_7VDy01JOv=^NxfW+s+Km-XfsNI1?;i+*HhCVzzR~{fk7(x;R5yoIH-aBbF zglSr0i@r2$t8EFfuf3yw1;K+5#{&5XG|PgRSM$%=L|Wso>=SNH-l@92RBAmrvX(Uw zEhZe9DC;_7UD1i_l}{=bEFWCQIY7iN61>du9inqjN%IP(!j1LjiVgCqA5(p@-ybkJ zwTOLw8e};mKM9ihgn+4>`EDnq!L~oWUkV7vRT*<@iSVtPn-e{M!MgaQK=wsm@V(c% zpiT1de&0Oh{~3N365^9mmBM@cU6uJ+8;$xkx>M1+bmz3Yf~I}Twcb6RSe`y|(uB|R z-G9FD`Y5w|+cEV@yYL*sH)|tpw!iY)=}fIEN}#IXTRVRIO~gZgJ!7-{>ATK*rQw@! zmlB5X^O`s>J!&_MBN$3bMYg#%)6ftT{@Y~I3Ewr&u4m776<#rb&EZa3;` zq2-{Vz5ZwB5~sHZ{$ioBq-?h9sMvAY?EHIyHVb z$jJJBhW9)i!dr89X-TTl-|I!>7vxEoqOD)^Y(>zC(dx>w_2|>T4K%Cb<4R$=TS)0* zqGR_;u1@Rm>AOdKCfb>ql?eANv;D5@&2ZEAGBk>!&avZd30Rw2ow-AvB&oF*`^W5X zS-H*@3-^9AaYoK7(q;MeRow{356H@w0$TazN))ijR4sj&{s)vA(+?=hf_l91yFzO1 zC0JWtC;yajcy8jZ{#}|HIyx>&s9O59;6p2;jCJ4{Gf_#AsmTEH~1Yybs zg+#R*C$Yk0;x&ysESaNHFs6yN(n%+%OOyLiy2=&L<(%1GO*=wBtt>~CXCFKSo-jEO z930So-|R{DKku`SK_?;g<*!O0zE|**wkxo*RFb19oR$mefxksEg{iJfwBOsh{};gL zus^_``URIT6yza{qoV9>^^s_|qLiA}ZfE;4JPzqyL*|R0Sp7wGBLcKKuOf=O_!>l| zkvcI+Qus20MFAkK`B124hTpWb6_^ijSE8U_tzcT}=9*QL2#C@9!DxoQCR=fQQaG{7 zUBI)Eo6w_{a4NcBKfX9#B5#`>$dkM_(8vw?pef0x7A;U(BqTq9qrOe6gY^puGrA=e zT2zoX!;ob0y7!GG31MSAXeSZqeloO+lY(naoA|sc4Dv39PNc2?htS9X?q5i=w0s|d z2NL1*xsDk~HpR3%;A&ohIwZx}J%`DvVYVFc8Gnse<{8AEs*ok9M0GLy_(mFqL<_rg zR%(eH8r@lz+f5y|x^U^$N!%+`8brJa5X7fB%Dvk@L;tVxSHfBIi&f(mPEM!K@%ZWy zvzFXOLv)JfXp8)t@0+yEt#lmk(>vW$8qW=YV`>Rz-{P`{>0%@Av~{cR#_jZ?;iqG9 zG)L)XyP3>i+TRb^uKT2(+{>F1ERW2P6D^;e@tN}}YHl~YG$Zqc<57O(T` zktAQuxs+os?Tr_sFX2_$qc7!$8Kqz8@0p_?!OwrsNyezA;L|ecdj0?Qa{=aX^F8(6 zZGh2$fX^kg+G{1QlwV48G<-z#wz@?7#vy{3dcW_@$?4H@$fbJ`C#20x$AvD3E`~yg zBHMq>&oJD)$u+n+GBRxW`f$D-U)#!lzc23m?!&US`o72JzAsK==YDV6{`{?Db|U=( zn2`TIZvJlB-|MpAaejAe+FW(~j$?`5u(|BL*B-9qt~nasj(5u5vNxPw`#QBgMYNX;^4n;M*8}aBW@ZD{~P$UBnFU|zR$}kxx{1=H9Nr*+GvCu zbETfGu|Yq?^TBBV+FyW=KqV-RAmciTQp6&Z2j}@!FvAm|DX*cs0R$A ztNA{^k%P&G@;!G;X$3)ZzxGj5nLz6Q`a_wJ*hlPGq!7_Eq2R+6v{rV9Seym`6qpCq z+7jCP__yY^`&n|gy;7WOgh7fWzR2A5oHN4YZHlw*C0YdsDY`^>&Nw5n2YmbcGB&15 z3-jJuN0!%@ajl*zn`+6V8wZ{;iC5BMmB4%OGr%pBFe7VtZS1rfvC=yqV#63&I+`gK zqYH3KYM<1u!A>*gNIE-u;-FAF_<1au^iYv9SDkp49tuL;pOA%=!Z309W9a|mUr2si zW_oHKtKk_=1q9ruwC;31FzP3ycO2%06Q{Q>L1ckH2gv(-ab(eYJ4Q=aOoxp(lS9Ot z`M1ywzc#-5ehI7Q_f5MAOLOO9T{v%r%bwk%yD;+e>hgm`|G6@Us`(%|QjQh2bLfeC zB}L#_hBu)VB^-`@+(8O;y1*8%Vfi)$wilx$Q(bW^#M>5pID3A5%KIJutZ$tFrBh^1 z-mf*9-XPUySz(iJJjN)_fpUE&-G6y0fBsDfPHKt;R zTls#^vQ@3@p@*BU;CwG%bs1lm9SaGA2Kkxe?%YKn;~U0+ z)_JpH(!d40zoWzb99^$0I2Vd8!srF=6-0oDcssETX^@W&a%OE5f(*O`4QEE%dpz4H zdt$BI&+$EzV|C=3JvO%e!D?eK;0g6}hc?RGuMSLq2RADGScx<#DhoZ-VN=XD}GUAU}~m6#oA^~fe1YU}=K5eB+THdLhcCv)$ zbyrexw@%vFL8dtAfu%rSsBxOnS>juutcJ5a^QFHG$IYEO%ZEXo>!5iCN3}S*7=Vsr zfW)1EF2t0TNqnK;Z`V_0+xDDtLX#{iS16y8F_#%t2qfzmwpf5HQbtJYa0@19yl+lbLe*Pzl|KmJG$M@H*syIzL-*XRTN~vw z{zLfOcC}c%W@4RJDG!#1VwFDN(z;i_^EbB>8Ce=&Ers(A+W4P!ldpMe5UcsQbMe}J z_M#^KLFPzI9b)@&n@$YEdjAM>noeVTi?1NwSY*sgU?8S(jX)?AWvGRHl7Pl+Pkwc6 zeLsq8umjhSD0|#mt(4cwa@gUOPz1j8Lk%}gnQY7VAm9G)8eP#D?_%7y{_eHVEe|wt ztoipkj=t(tUZwO)c@{ZPWITOw7lo!&)2p!$c0W&h_*Ud=v3{^1%^=-i zj}{?RJpj`8>oaci#QNP2+V1P3Gd8O9p8CA!TY1^((Knu+GD5NkKZU$H~!1u#oG zxx$g+(RncW7R!#Lj_me2SHq1r)IS*c+nJ;#X2BufMmgxXNk$%=ODGS%>dT$upSTm; zGH>Mf0`|K}G1)wCSrs%fnzGhO5R~ykOe>#nd*_V|k-hc}_7iP`Tq6FXtGJtK7;9`5 z@|B8i{RS7YaE`oq>r@lBriBFEOO^1rO&FMC5SzIzgsR)tj6&6&QkxPf(+k5_ZjEWx zO2A#!+rQ!#4Z3~kpbeZxu{P3(Gx7~3A%wWD5Qktf3eB54|1BVS>*x(Mg<~m{af{*|3m65Gn1>MFK2?yx6gQ5T_)*Lyjt zZ|s=Z*Gcm95uo7sAk;JB5?%qBG}I{L$Jk_n;M`=c)MDA<7;^{rp9&^W}l zo=`V|xY>4Gn=Mm7U@YR#1D`BDrNRf{hNkt6Up7B9u}ij8YN|DBG||@b!DP)S1KdX% zZYRa)l>;*6QDS1u>JxwDAsiC&4vYrmal(Oh`{1j^3^6s`6f&f;M6jz_k6)!G*xcNP z7>(SF+QRdn zi3ok%s!O}4m(qS8Ce>-TkUSA7?x+xTz{jT{A6Yh!+(~UqFcu~ji>o@!Qsx1iPfS>m4CMzRz|he35Gs562|g|(0Wem30U4ZyI?urs}t z4EpvXXAqNd_Y`<%8#SE=s28{-+Qu%!!+zK(;c9^C^1H9kFjzcx1zBb)DU=Po{xKTC zYDh#c!hApS+tKe1)*CAVdY9;uU#r5CsKU%;(E=41J}X_fOr z_J3Z$@V>mq^X1Y|NG1wWt-JsIJtl~JUSm3AqTkmUAX9Ho#R*C;oAi+c6Hpm(4d%PR z5G0x9(?rZr&?)hxLtIL61>X_JJldZ(g14+2=Oel2-_H}5y&fy>Dvfasg5VmX?Ae0) zY`KX8yfwH?3TrRI@~z(i9+T$Em!8QYb~8ORB?I@X8TEM%=$|dOjH;2wZmZ=GPNppe zz$@LdryLIxpX&d#%f=asi1a&W(g=F;UfP!?RU6#qzvf-|_Lo?q0ii8pXJN>L_cJ)i zIA1o2{4W4GK*qm>S7v|O(rUBceYNgg_uA{a-+sy6%Yn!16t=dt`XN@YJNESxsVt1q z(StLN`JtT1`MSElKm@KaeKXuNvmM>ooHgS%R-kC?`^@EYk9xjy2Q-GoF7{79x-i!D zYbV&0>LntSb#F9{oABn@$A~lATiuO>C@R9D&U6!5{Hz@tVtq1x&*{w$2b2QUy10&P zZz}WCnMIczeQy_U3jW_fCV$V z1i6WX0sBB^dh?ISak6)xM_C;|>>cOFqg5=0Ry+ovw8`#{^`+MB?P*Xg8t2+$F@0)J zrjoXN4QIP*vo~$A9GgKGb|5!>$?nv%4ZwdR`g2@Tl-5VJ%>JT#hl8H+L|Yhe+iqX} zAdt$)ZIA!EVYt`ZpZpI*3P0KS|Hj7eNTt=6wNzF=uuLQOORpYEVbq%L3*$?gUI{Y1}6vm!ihyRBi8*I^TzfqlM(cG6^F0aYIpJ3-Gf2=231jsQR zz5n(CsOQ-O@%>jmwD-4vy4_y~j5#seUe42Ww2!ufX<%$vv+t55elkOZQfEfO*FrE$ zYdC@fx!Flm_fy8>`-G#-nm8h{u!bl~>BQ4YBZx9`oL19B0lr_TIrk%jQ6D}IM!6rI zlU_h{6iKEnWV7ZZQcfmSvn^74yRvZk+-%YE+geSe#x;{dlA9A}*GVN)9mN-rK=V$& z;LxX<+@&*9a`5V*5k-IbJ?O+i((AW9l<+Bgu>lwxW`VIm!bxiTuf5vezyHVWivOmj z|NMVsEwcjn(JarqkM>2q|ET9-+tE4270iKB$)rWvf;EmPO`(^&Q_F8^3zyH&78&2I ztS*|9NEWm;$9u!NkM1{L*CM8OwgOTyn_jLL(hF>y+qBtYadz~|W7;gON>`sFU%QO{fvN85p}yg8gCy zvHreweoMSlj)QF8{h=}2N$wy%W!*h&4vY=Qnckq;sMJUUS!0RzuJ^3ima%BLIH$*I zE7RaC@htOcw8gKpF)CeBu|(y^jHmU)`T37>R1T%dnXz#td&Ryb<=)MJr~Ay|%xnTd z9Xp6E+`24t#qni2R|fC{PJ$tBh@NvhIvDlwgRPX2 zxD?ndc#jQI9z6xy>*&Ik{wy$QC+l6BX}Oq+g%G<;SwoT?4g>dNiVl z!$Xuc;G? zUj)YPv%uILWYpu!s~bM=zCC*(P1rjYt5S`|#`LCLn7zHb;q$U;z#WbUoRV)K(*6xR z>Kr&rjT~QcczFO^(ufVGA!nM`;T>A-_n6UbAoRPwwRO%mw`jft`L49UJGMIPSx9ZT z|H`!%51DFSSg$cWT(flCHrnfsvd+~?<7Qw|&A*4icBdRfonyl02c4lln6Oy?=>Lr^ z6dfV*!Q2bIZDLy2I$7ZFFH`Q{&h_Jdbh{VghVW;XD*-p&5taSG`idto28bqm7Rm30C);R=F*`?t_)oLD3yL^wj~<4{m8B!&LF^ zOc61!2))~Zyeh%8(Cn1ka%_MoIelVo*0$pOd;V%J=e1eN)F@Ys(E@rlV$8?s1seD*A7qQ4Tb2+7&rG4 zk5jd-3?`gY#!XDDB4gF)Sd4#M62LEC2)$#O#9rdqX9=M*aw4j?f_LXjO7R$hx8=wv zFg8qYgLt9%#kh>x8@ha>iOIaBH|lyjQ){!av$=B2vDyqtvMw2WM6Jf1td+2RGK5g4 z!3sGm(WdQ1L-Eq#vEpU%&!#qdgPVfFmRaM6eUJI!u+ZpD!PyOR%*pIb%!!7aY*>sP zKFE)FV#wEu5GBuJG`WL>jqYY$-qOj5e50E>T}Cku&9{SWn+C>)(GkndSV0M%+L@%2 z;7+R5*dw~+WJzrX=9oMeJ6o3xa%{TZH54rw9xgC`wzG{l98z1xyt$W{*tV1zoQWQ+ z6p6(R@kH~>!+W?%NJRcBJ>s*TSx|fk2iy~ddz$x{EeOBI@k^T!bU4!z=C}LCWfvxq zMZXN;Jo>ynZsb?Mld>>S)!T|A)gZ&&F{;dHsZ|DF@2#nM56@L2u1 znZu>prd(PvCQ<|1Uz!$zRX2n)qaZa>-$c!D>UE&txJ-a;dhGD9`|+}MmKg+KY?ubd z2J<1u;-Ym2fR(eM5PJ9gje$Rj>-h1Cicq|&b(fXW)`FR;Y(?v%fV>WGLYJ8eeDJXyZh7*=+drfD#Z2Uq9;nu=EP^Vo#TM%#|ty37epmR z=L9)J`Rbxki4&yJY+_-LBJJoxYDG_9kG!TjO}e|OymT-cpMr6+=>U%UT;(n++U* zQ;Iy<>H9gICA}p{wc)rN1_PH@3r{KmVY!w3t=<{Dt(Goktq-g7zwT?#`*LZQnx)>( zW~tU}i7x;3(X|0KUab!M{^zY_3iSN6>i^@F%^fMaI9yMX5I$3rpPgDGL60TJ?-;6) z0M=cwLYog(XhY{#?z}w^7!-Ol`t;1GXMp(GXPN_RKx`~mZQcsADTf(!J5^vN?<52` zSK+Fl(k<6N4nRhp504}^5?Kx{ElabQ0%`j*)&T$0aemF5q z>Ihez(q#N~1z#JH%}-CF~>OF5|POtd8u9`tXtJvpPB@y+{I@IlXVE3RvI2 zb=hxcdV1c?6tKU2YbU#@-kuD79o6LIEY(zBmtn|y>gw5(RFfQVy*cZlfcRTxU7v@nduR-;jB)}adJ`Cl_Nnk$*|@1f%4lgp1Xy5Ed^VwQYOUgg zb-b=zA?wy~|D85fXK|~1DJ3_HiXJuS3ST_awi!i(!^Qgj(dO6A5&Y^57t=Fl(Yb*f zfu8~y#b~J)2>A6}W~(5Y)}rG8z2EpN+26i(*l)aVXH@YPBC%OZS$%R&BXPWr-E$}3 zu3GT)&E1`rKYeFz>6%zoY@*^-lIu>su*EA)Uj&5m;?3Lb1awrZ(`M%Wv=&dk&~<0> zi@aEIl|yB#GaYRg1KU(A%q@ba!e#9voM^&>&BN>66f+qT2LpWGA@VVDfh_W@=0)Js0S8@o_-IbP=eJ1gOO2i z7R=wYl!$KiaI1#k#|{d4Kx8%j%n6F-S64$@5~9$qh;2L-2MbTk2u6pd`{I&~8MDg@ z6=SCain7_n)Crj~bx0KQTP@x(nYWsARs1A+)9Nbdt^_vPO@to?F3t=_dGX21ylTbx z-Am`hspC7sw~=vQ$HChAviUeMqR z5A{2L!l4M=V~+COyS%6eH$0^7``xN=1T160tk;4kFWeNNaWW|H?q}ZT0#~1>hzGAye zV7n`9?cOY0aAelqVXNm`>%min`gUyBVDxj1MyquV+!m?}?g+aF1l|z5=pIa^sss6_ zi)4$%rNlU`&d*y=%NI+Fjm<)l!*{Y)aQ>>YV8uaIUzn@Bd>LD!F7(SvNj$t%E4X|W zcwlWeFaN`(%YY5RlVDZz{`UsvlmkNpCav%K2Ks>6Bekr; z0-a~LM+rYASwk-#xF$>Y?hUHp1jQ03=-MkJQNGtI=(MsHl5jX+!=-yR0Gnyh?fhwi zA5S#iZamkjZ|4BD;Z?hqV}lW1QRtEf3s+gun|MBd8blga=G5tHtkg|uNGznm;OVa3 zs$N*>$6a?sB&0tp6WmE*95!Qg5||9F3>5jmVo59gmB{RDlrO9~jfv`7L$ z8BW8Jf4W3qd7OhwRLor(B!N@_Cn1F|TtbL+AOyh^q2_x#(4(MNm|&BJ1|>psZ+Xhk zxAqgSe&j#=0~{$e4O=~kD_q|+>A2#jni-9&!%TXtS? zI<+$AM zRVr4wOQpQ9QSk}?F;ez#S;fCs&DJ=Qoq`AtSX^Et9;Yz&c$QoUA&i@f(suW9Q}Pz%m8`zX4lBjtqQa`9 zLse4KSRGkb`$)m9=Y(60%FyPpGoA@qmw>IvrjlXTIZMJQkZPS~$ zH%4bzJuF~{3Xf^yMYjoJqIb9PV+49We|O06+w^70t^lT=A2R^zb*4Y*EdN%A2GCa! zmLIrX%dQHcNg1(8;{Be?mDH8-ChzQ)p*h4TP7*q_)5V@B6EwP!0`qJo5hy9&t2bv= zX(Eoer;n!<79LAWK3!gFR2)iO z&y-N0WmIZFc^Z|;NSSgPHCO422Z(!Zd%$+t_Y*E`r>!g(C-hbq6fplW)X`zPyJY!( zc_dw0-&Abg@q^6f;&6g!fM9zjEn{`uS4wYGYNdwzs*Yo;&r`YMfAjPAk{yVQudxz! zy4mtmB$+W%8a3E25*M&;efLk17@8Xv9-J*EVwO4~VJIoJ{_?@z?ui5Lt$J4nQFKz0 zD3u)<=tsZ@IIZtNxCoQ{r6D^)GLvu!W0&v6WIK+(Fs^!pV5WP@vqjU9ZksVDF4%Q{ zV{Qy*AYARRkN*btL8G1{yI0;TX}eYcRwR@aFin5{^{`b?OaunUjR@+M9ja>7PgkTk zSRez5;%ItkTjq)x9oW>JAK$TEn6d;jZj>k0!;BF2URtqWB0+YT5mU+TK4*LU5zc&T%- zO-mv)R8|w3Nc1nc%~e%|dayCU)APW=Rr4*KU1}gnH+R8@u)KW;0Y3gegL^aD)j6tN z4*2R)lvnCJ>O#!wYubO*-R=~AjYcItDVHbaBL0=lSfr_AJmoYV6k_EQQ2@8xYH;$H7X7iMeSK# zuZ7ieJ3uL4P!mE*Li%lEVz(bz)s9RI+kuNy2h(YVFkD@mq-Rps?#=#kD`(&EMhSE(%Mm$O`Z5v{JW90hvy9fnF zF$^h)(#elaa&ip6I+kUXrUcEkG0Ye%-HdvT+!!kv5dB=bATog=V2BOBl66ZvL9bRI z?GBtrnpnLSU%s?18vxgr`r4_?&fd=6e%GdhJ!Le|OVEX0$Omaa;?5Kc7zqqPWcpks z1JdT*je3pTeW_X1oJ(0sX_kMZ9(aRvI-r09!l*12i9fq_wg7oh=c=awx@38d)aiD1 znQcQ;OzF~I+7r;@#Fh$sQ|?)i`(#JAw|8MWydL4@)V?eB7r0C(4f*e2R&+AV~^ z`qi*Ct2Cu7{_iTY&X3`4*=y;(oss|dxjFFSQJY(V&{0mftYdara%sZfN(i_ix5qPM zRMP>*R(=?apCUAezAntIECtMStpXx@(?1d&043euFF6NmbB!iey)zklZ?bBtZWLHp z_TckSSSeQG*)fBUR1jnfQiWa^*yo{8JU&oE{PUkkV43)(3R~5E+ES;`MNqcDRuTum z9($@Hc-Yx_D41dQa;a}r+)_S?&dsFf1D}`Q*SptW)$1%S{%Ha7mi-CzCSkA%t?5;r z_`AftI2>QfMJ?TY93k0L$l=@r>>k{24Pk3J46x0dqYFK?9cz=GaZgQgi$o;qj^Tby?c!J%)1C8fdJ64i&$fd{!Lu&-)Lj{Yptn&Pb2pG)n4HA zI-c>T|3QNNy1VnY_Y-;5*I7Wnt_v?Tv~p!b3G!h4**|Dz1Xo%&0}pG0Dwz7ucq2@! zyA$nkhjwu!I&wP*4SBQp-C5-`XkRgnCK7qH$EqkS3fZ7p;sbkw9c|yLr2|_R zX=620Ef%zA`891Sg6hhp`mEGs?AbXkG_rH2X#VSJ#jZT!*^N-jZoT0M{~ie#@q1Oj zP>mLD07K!4DF@PIdRqv>HDDAF?Lf6Yoc6Si>TDeS;Rt1*rsB`SPzRyWb{4-o|9K15 zH(0~P9gnyRvLx-VX_k%Wn1FGb6+8-r0>vCcDCOm0AVe$&EtXehpk3N&6a7?Y#`K5o z;y`hb{`9x{FB20DIO+1N&b^Jp$dex&h-x(YVh5`ALDf}`QO7Ti?F})AuWRiJ(Jp+d zy;l>B9bG&1m6!EC$9qQrFR7#M}$^4xwK&nUOQ^T%rzbXa!X|xdy?Q68sZA!h@ zzpQLp66%J|F1Xat`bo~R!9irt_ZMVgy&{tTYQpl6hh&TM+~G3k394a*>M^ZtcmS1J z<7-S$Scp8?v>hJ;!owkPa*!y)IxZWU*gvd*>>gaf{wt!OO=EAPu@*W(UQ2tHFV=M> zZJFu&!(cHJJ3?WMif$2C_Z#m(B-FeDikfIq1kSE*Feks&kZ&@uYFNq__YZs0i`h_z zy4aE(#_N71(n!6Z9?oe`+xQ@*Zn|&z&{l)&V zR}rj0USGm@4R9#5q-knK;z%E5=9E6lni8I2mc|@~`SZaKtz9S5*5M8}J}5u?5FJ#H zL*3&jTCtv6J4HZfS01i)Q?|e#9|v8lFI1ydn0>|q@eu*&kX_#iofCdrI>j8vD&6*| zMoNtm!nl(#CGAYs=7AZ`Jm5Q;^a2;@(JhCp;frLWJ7Md2sCrc3V)u6%w64xtmF8_M&-Tyl!^VsNLTCG}7kKmGn>iokpdWyb!mD!LM_?33X@cIml%y4|-dJTa zfLS(HSd`^fE|KQWGS_!qa9|2gDz; zazL?zN*q$^urf!KJK8G_#`VhklF#dDD~iWo++G}h%WFFCuXjCTF{V+M9!+$5pX?uL z0dKIeg%CXO&?ApM@zgWVz0m4qk?2)k)){u#kO}K8;Bx)acISWX+CPfA`q)%3+&KSE zW8#SMbyS$|`2F~+RNvnI8z|JZ-(@{#-I#}EMXFLw$Bjs*n3Dj*m2xYf08 zRnlF9*hd)F30pze%9fUN-&l)KCY>7G($a}zajwmF1U{_c+&Q}IKQ0_@OLOx$= zSSLTp90W&r1kpX<8fYL&a;5vB2l(K{@QM>LQB;0$SM6oCW5?nha;!4fa!YJ6A=-25#x@(}^bG#Zh%^wU*S z@G9Dx7!uLYI2npSDIb(_QBPmK^>8%lm3u3G38c&x){am)XdPkR=mu-SmzFi}M(8sT z7xFG6zrgF6k4ywP>YK+jmWpB%jkioqPJ1bhB9gUJWD9H zun0626fGIF$Pwtq5q=5G-n6PiNY~~}78S1yZV$v{Mnl3F84{zG>-)9Z@=}q~D&mi% z?fdgbisXZAJV0CF5foJf1&8z4Tw7ObFxmxB{9#!(cwUa^)z%n@ewmhVbr+yHUqlcv zXY`X=ABdYLyP1M@<%=k4{wcm)6 zWOF<-Axwro08>LLLkDbv0g&!|fsKmj(E9V;E{^+Mq?{SUjd~O2TeXTzgIgZ$K#b6$ z_#ZK0u|@V=kFGhw544=9im<}+wVZsufY;tvq7995Lx60e9@%;(-5pI9)GdD2Ad-F< zbqbbPHo>8{=r-3|jv_(|5hl7-S9PDGpFUBqbv6U^ZGxVU3K}q)(F30E)t1pRSOFNp zTZn?KzynQd2jH1~BJ5e(lqgzGSVa`zlLRo-Hh{-MB|BRBejm;85*{{O{1Zk&sSxf} zD~w|93-_}j$&kt&<;5j?rui8{vP#(tZm&(crs+zd{a!Wvc|#NnXr1PO-Znw+MFsQifM3fO_*=Oh zwfG-u^}+pU zY13z`W&fj{<7s2(Q@I04_vsJT_c;u(KZJ!f23O2KD$a?X2Q3}~btt#@PZor)jK~K& z-LtEso8uVi;p)Nl-|_G4=l9kv=d11C3fiAJ-bK{3+?houWZiXqP78J%Te@ujN1a;` zirHhp3Ego6=!+XjzKes~MpB4yYor!V;vPY5FsQTqJmUI^pGC&4D$l>YV&V1fi!i^W$8WX8(HL9&e}t37i0$66WJ%(KXQ|G zw(FeNrLFrY|68%jf?^@CkXYnf)L67x^jb_?oL2mzc%=BC_>VHR475a9=2>bicUm5_ zoVGko{i4KLv8@uV2CNQSomDO>pQ>QiSZkJbg0QI|hrBdr`U9aP{Oi$>H?lCu-&YT;*Ni}&!W5TAvoUwP?V;f}?t<28ZGj5;r zuDBx?<-|_z%3au{-RV)!sBI>1rg?_unEf{UUG}fIQS)$=YeFldn^_6W)pAl;V`{RPI#g^xevHPIS(4Ryk{&>z(z^UCtxU z$DA)(Z&{yPzgUI)4;!dq309$lQ{X`gD4+tAV22Q7pcl}Wf8!nhdpt6}Jo)zIuJ7|@ zUf^^6$CV!jXcz)H@ParkI85RF76rwTHY zkCJqeBx=N5G{qbbqadb7b?l2hk&TsfBu{!sHR)`^Q#=)OW_g~=lqH{JK~B#*bI)qu z>g}r+2?YNNVf~K=ED!JvNDL^3;-F∾Na#*TBTU(!jF?9zmR-!C*+RdvGrd0&{`| z!nipr`K70{m0;mZfs++H&xu<;V;(1SE|+r!TRFhJOmm?gm8-v%s6s7Dh*U=6WlK&) zC>hc(wUUwchV19Gxwd(0Le0hY-uAsswcmCJy3hWX{bWBq)Wa7MBv?Wsi2sRLiZDai zARG`L2q+>PfkV&{v54&U=V}XL9C01-f7ubpIAjTO8u=7ug~Fk9sH^B8v-K?tH+E$tA{R|y7rfMm>)3O>5hn|XxGC5boX2=_Pot%TcGfy9LC3I_y~ z6ETjn6|x|L=TPnCWcZZJ9w9&w8g3s%v=2fS00<@u6cWy^Py{RrY?t5w!Jjm;{u0CO z6R%@5R>eQqFP3f8xPWJG`dtlPnnvc`7$0s2{&Vf>blg)Vp4wlw{Tt9EuaE_v@R0~k zANvIoOQL%88Y}la@W2bbGVuK-)=GTz%F5$oL1$nN+)6vc;bHG;{pA?4!)>uR^owM} z|98H-NSX}X&`959c2sEz=NJ7s0461Kr5c%rzYt6JF?6aopPF+&a{PxbPZV{?Y=^SG z9dJBntAABsR@l$4-454=YibT_2oQ1hErC2EO_IE*0CDf_wEk6rdR7gW_otm;Ns7!R zyK1XeWN@8VR*LKteIGU{r>@Yrw6De=4*kcQxBz|>XA3!u{c?Dhy+kv;I~GzPvgOCh z1#oCbKrm)zxMtuw7u}XC5Fy)i$*RDRv^mYBSVc&tAIu8c$0HU zdnPxrA&RngA6s*U-?`_k*u~9^*52^PMFyB*EvU72jV{f;)el$~rUXsyKh|ECWnU4< z^LC%vQ?gpf`!prC`kM|l0g34KLZsVFLp}7@K-a)o{QvN#U+lXdhqgYZ33Pu?RGw++ z6z&IpG*(IN3xN~OPlE+kKDH?SiN_S*!l>eJc!6R$p1=BKR<(#HPb_$Ifwc+_xOQ4o zOax7V1S!`c0(DkZ4(%Q|J@88lYtzl2?c9ass3^Ywhzo8u%gm{E4vXOy)7ewRwXv6E zIWoyZ*6(NHnxSFahjLh!j>NXoM(4&MkA<}XzS|%~82|kc-54u_ls!xC(QpP_J!7q& z-&}6@*U?P|eD;bHEP(G)nv1GciUVe8g{iZ~M9YCOYTUa%kAmlT-fkS;|qOb}^wqOIFnEBb~&*M3fbZE~b?e8n!TSNsx_il^~8#Xs=4B8w{( zS72S^Rjbm(EnQ7NZvzKJ427^$9JEy$X^X5x|7RD#CGqSr%IBljMX)lHO6Y8{jqSXw zBRY_nvuR;s!JHr>Hf6gJyHhxaa^&Z8jT9=A{?Ry@f?P_xKm;nCA;)aq5XBuR6T>v9 zt$V`nGVa^+!Rves5om~1ac=p)L$5U|AqsU^-D}e+sN!#rau)||N3r@}3gQr^imm=) zlh97Q&OZMB8!3U{w?q;xgfFJ*%E*cV^htt1l8O2!7rG+y8|}A=td4AqgP#?ChLMKo zD0Eop0O@AfYC(q}ngj@5j)OqP0X}ayJ@o!Wxdl+ws?xBv-KICSj51r;4X_I1UYI-+kute1`I!B2{wVdPt+Egjky9)Uxur5=7Jq#pSuxTQJ|riiE=$D0RmQyRzX>KUEdA zM18QdlJ{u0WwHp*_GnBAh-_29lfUM_dGPx)Prq`}%_$uOcoM_twG>3yppV}iL}R2D)F zFv(AGTZx9x&DaQajukV)cOSC)+#VZauI!3H-2bUlKAkM zrten5b+Fhp_U1&|5LhU3kcNObaa-{$;z-_W{YXANYZsUhXQKv;{&G0u1rP}#d6v3N z_^*L^bIxDx#$kj|eKsZuTMbLG6)i`mJ8W0K!8gPVjXB_MYsry4t>`IaOxj3}o_KI%r(_Vp=D7ik{ zf+Q7P_%PVE-;-dM0g2shO+W8~Tj4<)+lf5wa6P9oB`%a`YZC$Em>k_;BU#4G!&<{S z2X@PSSqlr%KJc1uF{XJNJGue1x(mi6f*{t5Vyas^NWR?@D^5tQN@S#%%Glm}or9Q^ zF$k{PHM6v%iHQ~AG~0_`Sa0tK`Pmllx}EnU2}G#hCE}W?Jr$;8Wij_T@69n^X_N>t z_Sm5F5q%O?Wow<)y>bvr!69IBM>o>OlAKrwcK%M_3 zpo-s~xB!ln6$J6!roCT=5UQYn`a(@c1+@{*ezs_vhJIq7Isw`x>e zyKBK?eby?N3i<)u$p6IyKs3=48|$}HKn*WDtJXY%=r*7)LeSFEZOqi#S^?D%>x1_| z75<{vVpRKCm+xUHl7&`d6bsekyp4uKXYuJ!55dJB@!98RbBeXqo3KWgwyuu)5F z9$2x}Tg!_!ixu&E9G1SZcHws8u|!R*Ri{}?&L-))vUkq&cl^G^x^RpF>LLWTFbTP* zTx}*-;9j3~{hig(*p&1g+Zq~oNLk+l`u2|(|4=EzpEPo2(4NU0f3B3K^V0EWt9W*M zYBW~5i=@Z1@pZ>-3^G6yC~3{@5`nQ5B<&J{@`z*$_@i{bEIyw?V|buP4^q2hv`zfi zk7=`6pfi~Jv4pDfjeYlbnHAV2xSP*N_Tc%OvOyU+Sdwxm5lvc6^C_u{9Zm4?`0XlM;J&ifbu zm&s3iXc`z$d=dp@YyK_8Qb{1bm_|`?kJH}ps9J4fpk-!B${LJM42OUmmr{$k?GWm3 zpkNa`WX5_bI2HeHv!`vK+Oo}fG}$>WelDrQ=?0i!;{}B!Mb@7R1f2HX==3O~a?xzg@35~^pTQg429Qm%jk>G|W0q|9!S^ot@EN?J!Zjy$A;)O}Iy^TLEs+!q6E24_PtEzQc1Wql%&C&b zQ(afeI-WnK>elgw>&>m;_HvjF{~s!2&%T4XfN)*K${w-jdg*0j?CEPsWI6iVCm})M z69DZcH%!BWth2Bi@mO4Lwy9i{V@FB&u$|6FmznzZN6ZrxDoOCR-lDv`=@%*eV5G%j zO!QJDUE6|dT2jGv;o|VB2V*5#VjUX@Xim$?*ppN+7zY1$4a&?{N`D(!lB#Ih z2ID54hzDFqq5Pnap$_hH}9goO(b8V-saSg27||S54H7^J+M<4Cm;Ha`Is0d?IKTp3yHlOy}ULZedp~ zNcGl+bOCbbx(gRY!0Pd4INr$6i;GkqAkNAMGb_$%@ z353$29tNB5BsRw0%tu`Phwx=_MM?kA(R31)@HYrRJ-VI(_8u9fjXeb|mxeCP*w%XU z?4sq!as|^)TuHRsLtg7!(UJ5Rd|PW#ZrfT>?JPw^=gb5plYxqC2rrMQt9InH=AGJU z8;1@Gbd%xC8HH=Qk~Km*|J7g{(`8)5z#?8av)mRYRwe}hw#PhDk@mFflFq>@F!UX*LjLobnk-nGU4hUN`ns;E zlU)-zY@~a|w_QFs3)&|&l@6E1a{R$3OfI`pzisOGgA8x3`<64XNwW{Y+LoeNE@mVd zWK^v%X$&P5TWor{Q+Llhm%zWa`WfUhMqQbm}b3g~d7Nuw1V+EexvRlqb~| z>z6F@?+OBTL1|9S(VNi}C5fNJ`$TYEe2vIfWt|8wJpkpYTH3_hOPd9G+k7k`eMPch zZfs%Ldz9Oe#tV6|@Ap=)pm^C7Tt%6Oi^H%R` zgd0RdF_c9nzKrGf{5bT5X%~e`jLV0VqADr`3U&<u{bOcD;@0cD5#KT|BLTf%w(! z%Auh_iVeC_O>G-&Jf*ftt;*@KT2D&kspZ_}fYDgdKO; zYlkwI>FNnDMa#6R6()?Kq|B~9(11;I-mz=YtKI1JO~YtjJ*SN{pBDV~L`FbWYBoP; z1B75kiarz2HLgsiQD+YNyzS61wm^fQ1dk-X!N5lXWRD}ojnvk+#I46@FQ0t`rm41v zkp#mlD3s{)KxJBdvvwfiNhSDFFqBvWssCPJ4Tw!C=vTR~Yd;MM4q4*C0pZNwDtKnN zpPtAN*x=*9j$54sABDa-Nm#D+pc~TA(d_H_sAx`H_Y)Zhx1{>{icw4{?#75>79UkS zj*AsHV|fq)VosdxoZcBYSd(6Ee=-a{8jnx13s&!F_${ztpElwD(Bt-YCSf=z?)8Ix z;;hfQ#0%^x)7~Hny8858lkRFIyMr^jX$kG~xcgyn zcJH0IrZ7QVZ9t*12M?_Ag-4;h)!J;nRI-zys1Oa3g-3^!i&(Fw&FzfF(R0 zzUpyXL})FjXhLSUYQ;LmOwnU){f_n%I)HibK-_)j-~$)K1nT0+emH=5+%CkW3wlJ} zLSokpx>cr!{j)R!d@C_)FrCVrd^E*~#aI$}d7lUExWf#$Pu;uwZau4Rsjm&{FntMP zc9pbDXl@LvIu3pbjU=?vPIIA`pRXauN|w=3@PR&0Ib>_7h6ct3%tRtBV&*OzUC5Xp ze=Rsxij#~y2Ux#T#GBF;HAQex{>|0;?90mZct3nv&<&DtrEa*mzZ-Rfv;tu*_{gK{ zEs4Qu{`|Nq##AVTRR&h?uvN?4U~>Y8t&n2*iZJ%@2P=>uKiKehsDwrBMq4-I*Q|zU zK@u5##kv~?N4)I#6c`d;Dl%DGMs3m(6YKEU|IRVFXGx7o0!#c$7t6`nOm2K@)|sM) zphFz{%&9j8+H8dDE*+T1z&O%XUh&8i${>*sNjY0e&4EJOq5lO zZy0nwkyBt#D#H)r8ZqeWfnXeT9~3TEz%IYNyckibxCl{-p`Ba)r$NgG7=j>uV&s<@pk6eHsRi%OQuxiu3 zhBMliOG{;~Vf!>kBdy8HlyjLtfRYfS=H^^Ry~)U(55x9zuvLkpU{1u5CH+=+fTvN( zwL18GFnOK6VlD%P_dwB%hfiPc6oN<}nj-bW$Wd$FZ3|mEKxl_)W+5>XUNiA~k*gbxBYJjKCx{omVe0kNzZVtcQa9TF&uZm>Nh?O8 zI;jY*$k#;<RxxvR!#M`BUr{MzUo~Z zT{lUrRkI*z1PZ(Iwg|^8;ImFBVMx~7$*y7?vTE0RP^sQJXN2H~h|lzc7~9fH^rxLnrjoxi=e)7g`G*K&qeGSBw57V;zx|E!nf)#@#x+ zQx>Q*K<{+so#aOI_grRmQAPjm3XY!1sc)O{Zs-;#Xp6QNHBb;|C!R=5g|T|FSto}V zlWCqz(&EE^A1>U4XkO>7^M79G)?(^}_I-g!x>2PJULc;VbmF7}1?%z8i@DU1sT_tP z4bqt4QWHL(9)x?|c=(%gzcyeRox1WrM6A2MP1Ow_I~152L2G(~K4r?8DVf(@>jdDgb0IG7ai|`$11P!RH@g^pn7xFq9y)BQ` zl(E^){%U?$Bby}!1F~~<972@5?bA|dV<1ezXUOl=fOF7cHkmrz*{F4gW@#8wQ#dj+ zTjw!XC=E7_gh)ENJ2&*!1hgmV=zlmE^a%P=w%Dpz)fe%iZs*Tv8L7Y zrWyafukI?c=z1|M`AuUm>6lu?Y4LKLwuZ}(ol+^Nq10zMtW<` zcggh*UzOe!O%qTgI4prgI@c$&*+LIm{Ykhx`8e*0!CSuteeNG(Qs|!dJsx-AK`u4+ z_1Z*lzZh$PE)3dnbBgy$^8eZ4=@lamMgLXunynLry}rJ70(TmWa6eqfWe%%h-s-cS zIfJl>2>ju!5Z$9OHoBBy4@>XjCkG?KI z1HXZQLNw<~0bnvAFI^E62Ys&$IbnC3&{TTgG&$zPgs;TaUX*183gysCVVI#G!zJV708N?(n?R0+fjLDZz6 zP3fSKvqEZoi)$_i1QRECxS>Lho37n0c*Pi={uwXqPA6y+gFifZw>5y5-XszUvrwRg zNC+aKu=j7$O0~{A?o3Dk1Gl{q`6p3DZlmBHN=6btcXPjk$Y>SyZ#Mn2Ex3Y|G55(Y z>NZ_@$TZ=KQGJ@`7qU2fg2$t}Rs_n5oV)Wv+vx8Tj4kvjk-l8^-CU7Z(Q^zE_jBlg z@zY{MvOFFz-@2U~(|yfgmzG-}>s)NWX34pc!ooaG95JMUKZ?eDyW(4`ku@5I8(~u^ zDcMuEA?|Wo>oWR_ie_X9U|(*`sN{_!k+3>b$wXo3>belxtFcV`nuE>;Xs5RMueFOFze2c$6Bd(=>`l~A8+cS-*hX<&=`et@73g~( zck154O4cJiH~Pc&!sM>Yp@2?F33Xj1b?Ca@@x{8@^sa>jC`n-hJo%b&Nh-hAMijo< z_i?9*BT2_aFyLw0l0VtSG zBA0J=oFJg_vcqKB?EbEHj-os}phFD4d^NZ%{(>dp8L=-SMsk4{fNP-|9zjwC!8W@yvJGKLJzLz-nGDS6= zwV68&9b^u($G*w6Nmy-$W|zKhuzqaG{#0ej}Xa z&Q$iP)6{+ID~-SXwXQ(}zhRoFyBhRdDvA`2`z|$Pp$=PIf#L5Cm`-`oFy67s=Sk;X zvaZ@;Ycgo%4`8&nZ?xH%8T<+6-!kN^I>&8G+B^;%5X?_;)!@P#u)M7phdzDq5DwMN z8(2hUU_-@~Y%6;#FIB_=d1>HdYbiibSMYWvaY{TyHa}JsVQ(0Tb?Eb=1;lU>lCvEI z9kK-)WdAvHC#!%6;K9G#7s2YLhBNR?fs=<~vOob5f{YMKs6R=+&6sWPvDyAjq235} z>z+;iVqAVF1Dg@Ne^yNLU@W1`2np!Eaedzth){^n-MCTlpaD%LnL7ukGoKp=LKg6O z5zuCVfDx|^O*Q(VYhaW!3@nyF+3iv?-}egBU=gjeSY*CxY|7+r+;v8Q4*G6054g=Mn zW`b<>cvqaQEUGnT7|2))ntG#gk2<;0Xjea7uVYeJbQDgIURdJs-h?V=FhodAe6`Y2 z6LbZxwEs2fKP>fd}gmst3ExDm-;EV(Rsf&j=Qr~2T~xZhlORnPNwA_3-3C# zk*Qm4{J`x~DhzbUp{ammj{~1JCj^nXI3Nn)8p&zPoh*gNl*--E_?^aue zyI65#tx*A`vg=)zBd*^|&8%aZzDNL%y;CSYj@lPjt7$bP0=!%tz0lOMwj3XdbL9E8 z#z2cgX+a;?Np+KkfgYkn2ofx`Co}gm4q?0f)OwReEwsMjUU-(mI}Bm4$g_}fFE&&w zP-PO!&$5#nRXUYQh7cTAnUzKQR2hAU0-b0&=N!zN7hF!|hX3$f;(vf&WKX9C^E=+Y>i9^nSORRG{Xp}hY z3%aGau80{*_*3%tYq^4D2Yb?h72v4Sx0^O(L+fToMuxx~-}MAasT@?m%uZyUWf*0D zPL_dDG1it=Yk&lQa66wA5BYvlJ2aT!_7ou7(5)iK zocHTDhyYFdYvV-20AI0J6U*>CJX}tM#`uQk7qKwO`^U=W);lNz2gPT=MRu6TW(1Qo zFm$%^%mzV4IdrA$B!Y+j_KpfQQw$r^q!1XDwCb2%F0`;tHQy#jOzBj!;#?2su^f-* z#qQXl_|pHxsKQ2;#>OTK5!+-4tvD}|Q71MtoZ25>$-pGdX!^WbX1sF={uLpEPzGW50 z099fr!vxd~cfdmqncblV`{HF;buXP86S9gYsjRBCzu>w?KOqyeLc3efWrqAwxk6Gf zcD;UfXOB$gXn&%2LNNL7_m4N3+OVmew~|g|L7SRi0BC?o67i}Tu>Br-W8|;swyZyL zWW{cpRqQx@&JHQSFur%EjJu-KnLKXIN-WdE2yTmQmcT&Y@=7aj>n&T%fNRaZ@gDEY z?&R#@`xXlh$<0<*#LDRe4S!Km_GQHdd?Ipn!lPLt4?|3eDcBb&Wb@}lbaRuu>z%Pd z|J0sIbR{CT8dN}&X`I{{j2GdjCOOjz(ZSS6UpcW#7IDj5Pj?*rIn=JGA`iDpa8IHm zb9NSX$}OIOl3qM6$QFhPOeNM-2?T~D^kMmN^CURA zB@h}Js*7ON%>-y|0SnxzNwnIpB5L(2lVzq}4#vcw$7ST1wD=6t^>y}55a6OBJQ(1{3nX*yzmnGJJNV@hzwiF*$*%+(6Lg(bdX)9DU;Odr_U z{hC=VXaCG!CWf&V6$fMo4w%N*v$*<1u5W7qbStx{=ZrMJ=gYCvJhU&5O`$6AlRUCx zoe^(YnSag^IrwA1@1LW{F)rw{{59CuqksDNv2T)cUZ#Am>!;8lK z_1+wc)05#qVQQ%Di%!!d*ClabHanNgWF(N@jn@4%n#tV6e49Uywde;%1#8xPyMrrE zRm|~B9k*O4a zHMHj*wkL^??clvTjQ?{nb%kPUAQ)eI$w7nn2_8RwtUXx3w_7+MZp^8y=?4BS+)QhL zH8>4PrN8ajdSm9mQ$hWJwxQfka5JqZ0$bBnc!O)3CFMz_VP2(loLB!=IFVfGll!{_ z$<&H_x{PW#1A@41aMilSMXoT+BUU9`8(e#`KJPv>qS4y+MkRECJ30OjTUud}jXzom zJ?I~J0zNdAXB+DKAtyo%-DB3veO}G`Lb^x~fe*e=FLeKRr^OQL2;H!22&+Lfd|5sh zff1>husg0#?3qi!G%SJp3ZIwF4P8t1>4r-ktf9Jsqrkq!>akHNG)m%cR?DqgT*ano zFgkr^YSs!Ch9)bRN~NhRqs9+1*cMOQf?TR^q;;dVxePq5+~+J^)nr!7WdiDVj?aw< z5s0$J7prym4+Pyt8X`BS>S2p!PJ>azJ5V{jk9PQs50ELa+PrL)2(?5cob{<;XtJzG zGI7vNYi*0&gDU489npu=JyLreR)m}m?Lw?>a<_9@hlL(aMW<3ptgjQ0%z5wn>{K=F z?L~QEUvBs=B=ztEC2t01CjRl>1<)_+p!_LBL4Qqu)XExu%^u$Lr~ zh62AhV<;Ea3l`Mta^79phzbZ_NuhK)@QD@uvubnpDrdGI#%>?`l@SJ-?v~VBu?#J9 zNqK1rJkkV*3b9{9&Diku1=hI@&1zVccV3L?iC)d4Vcy9Xw)D}4!+@<|?i+Ic ze`YGK4i|bwbU>cYC0Wv5BskNOXJ(j!lvj9Wa#o}^rG-TkOS*3V%!LHOSPM7t-toXf z1|@tD4%oeyy2_S0)G{enyXu<2PV4t;HB4>#NTXGZsY9Bh;~9}dc5EXu!)j>L1EO-FMKs^xne zl`AnMKrzD0{;``0`Ru735J%%$3#uli^$cV? zxql&UoH(8odyVJW_X-n_42>#)+nz+}IO^<&#rJh^TJe{S8s4|~=w+Drx9_(XWLkfY zVY-(XTK8F&SEE3IKeguayz!DGH0VSvD6Mq4MQxnVx1b{^dRw86(LM6Al$=+M?JX}x z8(pDx?n3G{lt}2Z_h1QZDe%5G*seE-ztU{ZTBkg^-MMJWzuuc3{qIF5U08j(`d&Ki z{9E^+#Wx5^)phwrm)tTE3k1RkSF@m*!KgZY*RX#s2zM@5qKev_uib6+$fi9dEUpwR_6LJ67{m@1!B3{tGZ8-Uly`^d4q<4Z=9Cfc`dYJGK>YsdZCg^9emp1^F@o-oWWm1^REAt1)>1-V@u9@ttiAxX8iOP3n zC%%b9PhAh2x#WuFqLR!pDaUXdlR}Xl(OMar)+7lf`wO)jXNjPWHH zUnm@=kpN2e;mtmq%K(SNpYZ#lS!Z#NMo})W^iioLEVWmI;8yOVe?JAf$HA9$*t{+nxCOWu$Cy_CnM#IVgEP z$yXKD4VP0mY~hDctM;2-_Ug;4re#H5>MOn!SP*Xo0yD)}!e_L7&C_ThuM6Bv>q>j? ze##@z6h_k(7Xk=*R9i9f(r^{51HWfr1Y;FhB!8E!d$PduBfEV(#{iTf;k@1UTPC~w zoz~mZvyS+G%stuMaP+T*u}2$oK7!b$^yA&bpT>W#)}K=-tbyeed&q~8YywEk=kYjA zTHWM)ubnZ*ii?edo&~8uxj42%THA91`&zbT*JZJL$!22U4l!OZl@b>Wz2*Z(R+_Q{Wi6rzP&UxlC+m~G z>08M;gf%Jx5#UY)Z`ukd@J)%(lqu^cL*Tu&%v_86d8BkY1NYtUc{J=FfO=mk^>4pZ z%=36ztHFK%`KjVuVqNkQQxplvFg_NjmD49gv%GEr)t%6*5AKthTWsq{ZQoJk!aRpg z&3F!E?B7=8sX3RqHBRlf)<0n$i{Rns538Q$AT=(6VLO+DNEJC8yVpVvxmE8L55TSM zzavDq;>~#=2u4>i+)<^_jmLY4_jW7C&+3+5nUgp`IQH@n9BEYw0SLw$A3l=}051Cc z1d-p!ED&47c2R(YAm3v{sCT&^-~ia$W$@A|TA(d@SBQuN_@35Gb-5rTHFgMr!2MV* z_eL1p&KY+5veNr9=B@)A_-x;F5B$p?7yFPRS}b{@@H&1QR5S!V>8aSV%@RS!M^l3^ z>-=hH@VOq}MiuDqL6t~Zbi7$F;AU}r4Vh9F45UF5{$rPxCsQ(QrL0&mEmCG zlOc=wlEBPt=x!}=`Ln{znR%I6o})@M-H~~GP0b=&p&bR0H3x!tBRysV?FmYbi#FV# zxoAR0Yc?R#hw!+Fge^alQ3^kW&1=b%!hno^Q{k zC?FKU{(sD*T{WhnivMoJAJ6(g6)_+xIlg2YhY1oxM+3rkiDy+y4khH1p6%;w!R=y6 z-hxzQ{m1{2B`ul!lS;e)vh64u1|`r7yGI6Els|U*xqAIWki;_u4)+|~O z=<&YP`5w4kkmNsU$()71{w+n8;OUmL;F+7l4u^+t`{(d;11kUhzW)x9tJPn;umcnp zM-lcgnw{y&BS#khlz+PR$QQug@z?3Ckp$#@;J(*@o??5OY|}@pKmMci*S*{BxUCNS z?uoCH=R9=p5B>&7k@$YW?xvDIUcc+_RGjMi5rRI`msEcmPv=X12zuqY90!oAyPzx6 zEW!70rMGO+_niB|6QJ_Gn7()YiyimAsYeK-iZn?Gk2EHHI;&H(zh8hJE4=iE_DXxG zv)$eq|MR~ql50J+eG4rIK!1JwPV^{zAySjEWOY_<V{|DNav){&#NPf&tN<_PF%GY#cc*Q|CE(jMoxOt3e=%1jNF zKO3HN%(4z8tX8{noLs486z=rW?kJYnC8?7|+P{H#zvAJ{O98j9xYp#$!Ym?j zaV8}PNG=w)pkR&-H*+8BEGN0FFfM1N*`7bl)Zq3L{uq#Am1 z&#ZoG2|bh3C(#1Xw}vR#ZdgUm4p+QaC|7U;fkvSKW%IqM^u!i z5qws)y`u!`nqM*Mse3-Q3;)UhwVImrjL%SAP1Y^5703C#&C$@+PHgV_D9`K1`G?(5(+%~N>>#Mp*#yN zUUx84Q+GwL#Rx*nOR6*}6dP}6_5QW6=464eFZ`qSmU{U2^fxW)ynw>LVQh*?= zy2@rn8ObEuz5@#e%UPJArVv7zt&t!|(x2gyLR>Re7Ik)ldAd#>kUGYsa1{k;j;{6; zF0jS_whpQmsnya9-zLZ^ho@u_d6aueIWGt;&4(s5bE{6vBalQE1z}Qfo;rk`2i>7> z(sRUjRH|5>eNN_Cg02!E0hR!JJxf>N_vhMe@6^d5ic-;Za!LucY)LOrf5eEu+ab*% zS+P39COQ`Sd)e-vjpUG!E{ZYrb$q! z?(^MR96;st04}HTu>>f59GnxJ(pK3Pfa;PPj8#$C~ug<<@!eoiBLJTJPK4w=q)P0mrxAdjCNLFN)lR4?)Tq>MAl#{ z9j-h>w!dvSi!zyPFoild=BkcfG8zLncHte?!8$WME+&w*p`qhFa?e@Pxh=qbAsbQt3gT{fPbTOh zY~psMz0uhgu%&k4mWs60TQZ_Sk@m6)f{7ET)PCZLSfNpFnrU}}SitEmE+L9iQRzxh z8&shx3K)CA99lvFxtLi?m@%&UD z_+ZjQR!FihhKjssd; zURv|Djn>6eIlS_?o0xC>TI1oK8JrzXX~DThc6)D-MKI z`y5w%5vD9XuSWA&q99c)q0*`>y*Gq|`JU-`>yN0~Z=jN95?2guVB;Asza zp`jG9Cl+(H+47`K%wanG=irMCe(aRlU=D`Qy$D+WtN>r3eQZFE_i3fr`yAJEbW<2f z=FNWSK3le=g$R@#6z0oW;}QRAWSKcfYhHz=@Qx@%DaG6G#2}sON|mq6M+EuH7q{Bd zqMt7}Xxu9L!?4x6cQxXSc;R@NUAT10S$Fnoyf1#fjikmpIwK{bu}XigJg&H+H+`b! zp(6di<$1$Wx&JDbMI;JhRx826BdB+vVlt&`B+HG$EbgBzBMYgi|1G57K6yZTYDh2P z%3Sc+2qY4VEXY{(SGUZNwwjYuJ;ji2drW2UhDk3+>k`r22kBfJ-_pHdb@@{*s|ChN zuT{%ZhiNX1O>e9$b#rLlce@T55q?;Ls?$(HS*(K{0~**Ba}UTZod*k?Txj$2}Bp{L*zmmxh(bCN|#D9n|xAg*kac)_Q6c?xsqy=g`prVh{rjyAo-LU zX^f{)!>Q_iLCfZA4Y>yVTA*LH(-}PstFo$9ZvI)nkTMrRYmxu14^nE-!?E%>o+Ovs@QW!O(&_SkaCo(#&oTU8}oasrn@ zGq8$Tm7;dpf_Z&}!3A{hIi<7^5bqU~GGr`>kSjc1jU|;33_Nz-9<}yh87yJ?jzuaW z{A2pmcu6gJ32I0Gl~c0^yz`7Z7mGsh=5zqf>%muLw}E7BnG%s9UK~Q|WhxZ8O|K!7 zLMM?81`)5vd5Gr)A)4U0cy21~FkZQBbDz*7Qm{e#bG>5jQi#oG2iuyxR)|TBw_V6G z7&|Ve6sS}9Mq$_QFDgX}*%cYIPLfZ(#1ah8m*woiba)Dc65Cv<5y=5ou%2_7q_XP_ z+3``8*~}3je%uQDm1o)k!w-~hv)PUT(HR(iH0wM=U+WPbjZ`TI9F#vqgJr3_%ZFKZ zzx!rf2GdPrR3yxVnM}(eBF__i)L1hhb;rS6(6Eun=6~52qHW=5)a#%r3blj!S`}O| zqUe9)N`r|m>rje6gyE}Qd!4uII0;2#sSHW4anqxma9>ft#foKZDRletGu(OAunBoa zjgMGu^oO7{wF<_XEg19!kxW|IDC^4Vl|$^Mh+56=MwK^*TwrqsS~f9@>?S;jqxoG8;4mUN0>s%UqJmDsrzmfs#|k@e>~%sq~kW z5%h?0{4Uo)_`0^NOEw$F?bYOgDOH?)K{&*AYY5guvsrYdLa3ThYZyjP5n{ok#&x2; zrY4b2$!!SV@JNDr=b$ve%N)|w6P{6L{h-hNQ(~4~<|9#_<+%ufcv*{5e_}@>d?Gth zetEi|%_?LJv+z`lmdznn7)Z!!Rk!MtQ_nyYa(NB|M5%aSDVjz|W3gzNcGH1H$LrPg zV2zSgC&?v4=P!2T1(6iKS7;gBu*`~4r<58NhK8c#Sj9ATZKFg3tJRuCwJoQ4M~A7Jg8tN<>W>sAvQn&|R#O$ppN7M(Nfk3O9*Q@Mq3?-LHx9E0 zcnjY=F2!P&Pav9dN~KEfkYITnfjnN_%DIA01KMxGi3axqLo6uWjUxqb$S3YEw2SCc zAb_RzY4m79l8>b^dwwA^&ny@C933u7xO_4eV6`M$tr+cg-z*86Q5N^B9a0j_sA-K> zJ%OZ-=QoF}N`-WQSE$PMp|m&Y2Wh9H?*J+Pecu1K0)F>3LnmqeropRw65=mgPc+eg z5%$5ouzO`iLHgDo57cG(XE%$R)X(3l>YYzb9y$d!WMcpsVJk`QeOe}S;xKD|0|fZ0 z@x>i`6MygSqm41l(efVB(Ot3D6O2)LfcyvW3zHaIjU$NkaR{sppCS9 z04oKLr2TNf9?rbQvHQ!1ckI*d@SUJIY()=@J;}#u#Z~wZ#XOu;oR9sA%W%D7A@(Y+ z#XX8Qao<(CUmX5QaRI)dxbTWzu7V?1=C5>@vGB~p7Y#dl|1q*P)qg?wshMPr@N}r6 zaOnp=`h(bhfYU9aPvq^`vr}N}*PXt#JAr@~m|WYZfk4{}8qo>1B(4IF;e6(Cl!T*H zRG3?RN00)(gLm`VuX+ed{KD30t&nUf(M96l4_Js)y`jM#{OimFtRzTf<5-G|N4OBO zX1NAk3aXHhWgFK<4V;%j{cbgifNDsKD+~R8D{F!|ii84d18emkMe`9>TB5Ln=!QppeM2f*b9m(_?HVBoT7`4!)4(D-PRV=2>Th;9uuLm=2zw*c zV^)0g1%1)h@tPYJ3pj#?BOEUCB-B0By~idK#C2~bi>PY1C!CRZUtpW}fPJ@kya0Z_ z9&k87&d(k@vM={z!|x49?un6IkAe`M;{UTga=oVVA>cs%P~cIEf+2z2 zmkA!*LyxKUxeI^IQd3ZPE_^mj(98_IwEKy=o0%2-mG15OQGZ12Pi^ zFe<>*uY7L1Ll<#EHD9B#1dAi*F$^;A2}35+Ls2Eorej211`9_MKW1bsF<$o_Lop%agnwS9}aQ#o<-w5!;WOu1>{kyB#6F#dh z=-w|XVY?A)hL?~K(+g}!#Cb3hUz1S7sFXJDsZ;R#F+l3M0MO6YSl>}XOGd{hC1EmM zPqslK{>aBKqY`zyBITl=&tnG+mtkI+R{qs+6nx5T){m@i&=q{ji%qT;1zIptybQ=3 zAycB%iF~;65y(vaUVBgM8xjeT0tgdKi|QQzF@(^NVACNiH6jA64IZX|pRX`lh6TfS zx!|ANd=k|D`h)()*az0QCLi18oe$foI!VwsIRPaC**7l9P&lUd#oUuXV6BVpD=Bs7humwD@aR2 z*Q$2GyQ$YqTk)4&(A(#t2p*f(^zIQszFoejXkqz)ztWOpHr5V;59FJ6 z9`xPIVFw@B6^CZzbz%MX5hJ8Dn*Vuui4YOFC&_rcT^FTz3TPFSc~A-k9#oU5JD}rU6g{$ za7MLyO-!G&9AxtOSP;@z6T9M>N(lvs3X@zls5a%od>CY)Myh|FFA$9lTsN}Pm_fSt z?1OiMP?1dxq^gTuI*^EuKJdwV3{u{y87?p{nLjWLu_597*S~y>_AI(QR&j0|)^6Dv z<(tyRdP?fYaKR9Ot?Drp6JB{DagF{R2v-09>8|$^YB#h9AuWOYD+s$}$&@mKb~e#8 zt^ZWhE2w(a&QMF0Vi&axHJf>T?}BRiPvyC4r|gp=t53h7Q09XHJ76IGfGF#uAC<_G zM&qy=)t*0jUSi1o5r|Lj@4G&aZV!rq1_?pf$-8j)-(&hLFA82>5EigQ@iLv@q!{Q) zFggZNv_9j21A`21R2X@iLbHz@99drT@G@`OLrcrb(0VtbLNPTSQx_&6?WEhc{QT~! z|5CD(BadhgpT|C4Ba3KsBT7zpyuE+fG&T#6LZsfJfsYWLo3!$s4fqr;f23ZA1{N#&kV+V&oTcZNAID}y`=C6H*g`BAjZP) zdA6v+D4~~*3O^&zAV9 zoUpdqK*HW{l1oI}Bbl{us}RvJ4e{A zE0rQXbF3K8_(gjz;xFQ!5O6^Wr8rAD{22rPZwy6eV@9Qs@bs#%ng5sQG%!{@M}r?e zUCr?BdpsJq)XhL0ir~)Lbzo^qt)5Uyq)*1Rku-9#F+vm}TgOwS{z2QOUhw22>HRRq zD?~qy&f9lmV4rNNBm?#y$A>mGqM^-R70-JRzD@>XUAf>!fejKI8<8l8HmikPX1V8$ z#6&kr!h*uPMcYaQqIlsBHHRn8c?b~KJz)2LW%xbT0>HqE9BwfdWgF1A+uF2uurp5! zzP&^ADTL?~x3;<#HeMw5^Spf)K;xf7%L+fKe^y2Un*yV}IBdgbt*7jaDgD@6Z`4Mc zU+)k@jN6WVCw^B?muq&sP>vGyZy$zaA?SXmh$}VHFBZ1`>vA`)Je!RIQFmPNF1hQ} zbNGvVwIgEf#86RI|vh8c~)PC~U}~{0j7zLYBzuhgE2~ zz!^gH2$952KyvMos<5hYu=>HpZA~wk%Z7EOG-seC%@U{Y<$nQO zl#FM>hi_jtIk$>5PkTDcqCn7s>=`7CnlYRq_tBh55RZY#bmv?XEJ55V4xxmBki#{3mVD@B*aZA2jN-KI-89 zPca5ud=KJmmehPz1>{bgua|5aa>rFMHA|k7=xd3@b8co)To3ulEiSyC}NuJS@jg} zHErc#oU?GpRWr9f>9??FF$#&#b|`sn$-IoaMmxnzvRUhDU(Mr`^|PEWTMQOnq}Z<} ztdknGyO&AuwWf4fM9g{&I}~di-bGG2kP%Q4XJ*E+H-yPJnVnpe(#}xypT9u27cd+jnt{=twju8qPvF|D@Udu@;#Z-)A z79M{Qj?Q*C%jR=L<`VX-a7gr-yb^CH99;@?JPA>cYV=dZE>=Yk)kAkWl6k`$xD2_l z1*#zrYPEcb=HqA$wnVr!3`C^@lxU`1GHp&rt3@~UPzMj8rt$h3b0V*D0U$**Gzoyq zBfBlNR^7QeExg{>1ux{_C`=+aM6O?SAd!wJM-<=Y)c-_T*qT zfwvRX!|}uHhqYHg6F>;~-txamDsjLAVbPTq5Qg#w9j28Y`wDi(}FR_%QRnz%BeZ**M-%2`Q#)|QA*DrPK zB2VgDayp7$m6$10Ded~zkY7ksrQdRxVIj+S~L72yn*_zZ@8v4>-N=Jf|t=pEmeb36f}iO zV)%XQA3nam_w;Jrey!Pse#s=Ni_d-3odyco=^IH&)XTCBiFYH*HX&I$cH%KrNfz6L z#=W#fVR$nZX32hX<(Syt@s(nMH{HLbP+h#Rt+!A+sj&u-F;=K_N~2r;XhlyNH1t8i z*ievI<(F}Zz}8NyY3b@jy+-@=fE`z}ebTw<2zl&jM|(B9-OlFv)X2-1hBkjAe$oZx zrh;y0$_!V1+t8^vK%$TlLdX zk^72^{wk$ZqUj}b_n(UiprB!3`@L%TW?N^2h$34b0kKFMmGl5|UQz1cZIdBBQSQnKWd`josKlvFhVC|w|JNI^nDuccrYd}-oO zx)3_$zWRhCRj!>;nV=}CsQVp}izH-K{nrLY0 zs&y70OO5}#=|*~n3>f)@W&dR5Z*}S&bKFQq9Ktu9&e><>DdqZ8ppBh_ldCeGX43C| z=`yILIH0&baJ!{7z{}UAh9Lq_K^R;}7||3VcJjRbJ|}s6fw52|ChFCvAJ?B|$@J7S z&%N}jl2j&FL^Y%&K>C{@E2^d&re!;>=ks%U`V`F3AP+$$`VwO$l%CpC|RzkC6M?s1e;;VpC5-Z zc9KwTCQ8Uc*_HjDgMjCp?y4F-|NE*gB_Tq1zGIypfmb~%IX*(fa<|AEv*A!^xkrL; z2VO%6n{JXefDoDQSs^q(Np6tHZ--qzG|gYYbHN91e+)jBd?W@F@Og-xpO1j+2{~|L zQ3^tY@Oa9_LfLtU5Xg+j za)YzLwdD;ucXk!yVAnUE9?CA@I-osM$AAv*MN)KIO0cNDJXPZ{tBq)Ma*M*NLvr|ggzT(k8&Diziw%}5!U%gf=bl;q6_Zs zSX-$O)h;HSDdB8Sqbp@anTQG(B@Z6!Ydum5O*5tvkz%ObZfpK$x;N#{v_fsKp4IAu z<+Qdi1ECN$uQgy>ghT!on{(I9ifOh+7Enu$Nn?_FA!Sa`=%d81yWRLK>}hLlWEH#@ z=h~bGADN%zNSgek=0iy@70JAx9r0SEx}&8SW|6k*F4_BT-+au1pb)VmkFzx&qYvuvf>RH45Rn z{T#+_5L&EKVz?A9Zuu}+k_Afu6vCx=kz>Q;5eBexV3$c4FidQqd!_+2mKSdT@tb^` zpal00K(GDaumnIMOs}y45U^9g003+NfbIck0B->CD{vLm$UQd6j|%Z-jLrS|$+2AV zcoKRRD#n#xY4Ac2Gegc0CecSM=6zow9U$}+WS-^`Wq21j)({W0zIbZCPvGa@T?O4MgT03=&6St#>snIr7UzT-&}k z>v>pU6xG@og7{IenU-t)AK{BTuc|ejZA%x*LUR&K(0^EM7~AFrThcz7;mW#-?oTFLTj(bCRieu&P`s;A7 ov+5Z$hJGB_FISZ`lH5O4PBqq<6^F{d-($z`vHZ{4l`jAQ04qYQhAuxyRzXrPKTTFaQbq<~KT}jeRDlpDDgcB^FLV(I3W0}s>KqG% zQUEadTmd!$Bm;y}1Rw>900)c`3XU1)u7CgUrf}UngU93Gb`^f*G-74Lgit=fN@%& zoL(ipmjY9ITztcJ_{GpUX;Qp6cBW!0mPg1qJ2FZekL^hEpklg z`pYI3H?EA0IaPXA9C*U?iC&(8Z6kk@^Ah^!$MMa%?~_X8A_)m#S~rKXS&ADsidlHa z^K*Ld|4(coh-D#!PaFsqJWC;%2S{leEO@XGq%0L@9o0cifp!}eqq2^s;#7?qHA>y7 zl0mI&_WsO^PY?#2AQ+7xL0hCi2@qJ0o{xjI?p2lAMnnt{LkuAaA%rA^5Rwo>01Mah+fH~c z*#~A_;Sq#Ty~>$&%W^j9_8g!gVOwiSApp&cA*3j~PRdu6*Ae)R{s2NhfRk{95%QUhv2ut3!=@x{ zs2o#gw<#Bui_)ie(R6lEyQq5qpRLlg{@#@|C+rp2-(2QzZo$h<5D`T|_w2M)Gb@-C z_O7Uo>$nztuMaUYl89$b_-9s7;yRu)j*&v7b;J>Hk^p9YP9r;`Ooo8JciQV(3-R!U zwuc517UmPbU<)LTQ^=S7^q)fz>U9n5h%}9Kecd0?`49iEIGLj$=r! z1R*f&U<5;i3WZQmLPUy$NRUV*@% z{%6Orn}Z2O)hrq)$W#>+_{MZdALd8!#lr%Z%sVRU2z-C~S?dBAb6pm~fiNH*gP@2& zRK!FqWXB<+o0Z`gN`MNhTMVj<=c7;!AlQou_uO&-k_upcSZ&HOTS!7O4pW6k+|U+RQf3`L*A4w_6fF{zvq?a%9CL;@hKYS7Mi4~P$* zE#;61q)7zHPXsf(dWbqEq35&=KVN0oqlKp02!YWxaEfq`D_su^_D3GVws6yyPV>r_ z7qqZ9b;`+{@xzJiVd#w^jC4D4%^8RFTBk#sNlf(S1hse}UYVYVkBx;0c*g^V>;&vC zIsDR4+pRI*jHDxLQisno;7Z=pxeQ(w9zPNCYjG$PS5^g2>wb@kip^hQdUPOby=&~TzE=mO3m-3p7TK|#FL;wV!N znPURE1n97(Oh-&rG*u|L=ar7A9Mm+vC)>_!IRvXj(}gOd0xDzrKyqAqgTR?)$Rr&o zG&ZD)@quPSvIl&yGT#a}X{gL%aEgXe4OnOnmBNO6_6&evEd5T9H0exdG?a$XaH>cs zxxgm5yx&r7AO7GvFGoJd@-cq^IEUQq?{5l}LK&+p&>YksJ>uUl5)OI8R)~VAh=%Cd zU2DWbHeubK1IKy$ak(K$f_er7KzVw??-&(-`Zfay4ur6fRX8?g~j{pN<&~6g*KvD163fpN}>J;8s zLoIsvTWN6H2`kOwkB>i@$DX*Cq5v5F7~o*=I6{PtBBGNGgvAkxeG(Ce<0u!x-c6$t zCtM|_Wy0}WzVRU8YZ$^nn_^&uOim>NBuf%Tf{RNOwHH8fOT@Zp|LjyOEcV!IpZyLv z=#ayX=yTjr$Gm{A;*ptuHL^kW9C5i5_?*Q985BK%;rjMXhWmryptz>{6XIZjo^~vpPLf^;OiUqj zEYvy@EojND`6Q9BzZ(iNVk8bnESWAQ3Uc!aI`T*`5nq*nmmrXJeM&+?^3Sp&?%{EV{wIH`aHt&}vrDMT3W@In7xmbn|L#-_df zRncS}@q>(4!+_*Z{n1fj9QUk1YnpR}Y3pMJ*va!MFeDafdGdJoW8pX*CvXfS09YNw z+aFr*H97pN1yH4v4`e6(*Ki z9z!i+kps0yg=FBm2BtRTZ`$qpoDA=d0{>tlxAuVF3foBY8~^V%EvCfeL!0{VYrWb(4g8C|`X&nl zUF7Em^rU*NJ=1^Zv)l7T)GiH?4p;}+4A^Z7;4a_|P*9N`L52u*0|S8FmzZs?9d3DO zy!M|e|Nm|B*mNtjT4#otUU=@Mziju94gU3-)xObfy|vy**c^A=cGDdjZ4Th3YnFM- zNM)4>MIS&Y!m)AeA}5Ib14#Zs zAmC)+jt2;|?Z4F^>av=|94HTj2*w|g3Ez+TBlAVj^H;|FwVaz%*kTs8(c zJS19sIDnZn9EGGVt!7AcwzT(^-=RK_boCibcb+v!mK34orWf{Q2@F-Iqf{FJ!9Itz z<>J1fh01#%O&o*Ms42)+M4k!o8s*3-S5rTapfdc4C*>c!_qWIJtobNkRruyldOL?; z`u9G9!8wfrILcMj05Bcy%XKR3UYB466Pdj1hq!NIg|y7~*t6ML22%)i2Q_?qlI_9a zoDLM`TJahj6%aF?MoMgFf^5)5HnMTzZkiz?Y*Hjn5+(@%8>v_q#NV}=>x?>NLm+OWU{r0?V7o?Dm%5IFT%}$}d3xP0M@oiOP zyEQ6Vos5V&wnJe`ZST-g&vE0!`28|*_0>WpKPul!d!p;3ub()1HBueM_n^_-M*K7* zFH+h<6`Hr+Nsu#OldaDJywUDqCFjqgA>2rY6?r+c-*emhz65JN52T944wh44`$v-U z-{Ltg7FcfM#c5(nb1qM6yW*GyDeehOJ&6WzEg3Wm7|SsfuucH)`@;&+75s`qc#=;3 zj%t_|DY^l3t7H*+^Gy-=pZa$(mb{6^;PGv!GUmRU;|&+miN-kF10|cyn82?go22Lj zb!}3F*w1GyKJmZLVkD8Su;?Wy5Yb{i9}{;9P!F$wT8>US4o$*(NOZAM`5Su-=;(ND z4h!ob2YMFtFRem#ygyV83cEXeQT#GrCGVq{Jj1#YT*Q5a`=|~X5u&)*(9f)?TKOKG z7$s!$w>e{K*`D~yIKq-okF4nTc;Qs4D|Y?gWRmNFQtxkMTV7?ONacp|+D z8vGV&x>0X1`GjNGkEse?{m5{?n`h!bh7WnGV_n;D+Yi^8GdqEl`T)kN6Q&HUvp|I2 zu!#{utr@(4D$18K3k_V4F*BDrU4K+d3X4yo1ZzkyC&$d%m6|HFGn1#}sIVLNBrSAT zSF|$VSXCJFBnir9FMF99{(Oq2*o+%4+uIX`r^_(zR5D;8}0pT6^j37qQaX%E%rE`@0~z@tm`ePs?DvTlE3Z zDNI&%<)%vA@D6{aRwjdlY-8{=96fTwN`2SX90io!G_&e=>T(j27=Ev%6;1G?(i)P{7o(Okix4yItmVpYUfP2i8ApYJnho)ahqsnaitE15V#$_*$2HSsy?szdsay@I$jeHe5+8lWW?u6X&o*_@}R3vz|B}3;55Bl(x zW%7A`LoM0(Rk%OfYlN)ZvLN;ivAF#BK>-0HsdPH^awjTD7U zX^MAd)V3$JrIjlo=DS104_l%%g7SKE+PrF0;4l?&Aa$~FPi zg+U^>5sEG$xr!CRfXHcXsw)hBAA47s4i87H2l%H<^|+3>U5ith5d>9}oY8OQB?ByNJLN-ot^s5Lz* zElD|i!V)G~j1Wo+@Hk)ZXM<3ar(1H$j;3>>zGMapn|pyE%XTS_ZX_%yti?%K>6^tW zUQW|O0R^lc^dp3q+jhpQ0;HcHi{0rYUdqL>Hn$=%x+0-s&Ede4t3rJft!VhgIa3b> z1`ic|F;;UNh?xQjYi2u>8Q=Rq5_GJ%#g51Dpi+JO5u*1saiJlh#q6Piv}TVcmp*&V}5UF&(d7C6&` z@NbeXn5*Q1l~vZRO8UT57TGtzXibpL?VHL(w+G7dr>D(Y{XnuNV$T5qWcg=HJ1T|4 z96S1Q4KvK3W5ee8SEQk2h%GB6&v(1{TrF7EJYd_mD< zAOPM@y1cA(qSoAZoFyP2VU6J*=X+5$7dbkkEo}$KlH78uhypwYaV`5iSW(t7)+NVII6f^@X?d_JvqRWz+cMdrZRTXn z-l1Z%fS;`n%~FiBrz~yP-=8lo=>k2%)3Tj2rCf=`uefe_C)Fsa@wKDjLgUzB3aPmD zn1)94w*?)2} z#`&%;v+CNHyI)$nm=$NT9d2#d__)fc5H;xYy};Orft$ zJs!HPBc$Hf)uo>A2-u(lltS;LpRQo0d=%?vYr?2a6Ym#c`?#vD!v!qaRmm9pO*aais zLtK!3Tth`E5__S~u{k3FRMxPhRwBKTuhq?A;6(uJa7a-h^{hk>nk*1VK z3Efe)P}^og^47lYjlf^+sy8`H;V?S7)@-$RBI;iE%6iHp@Fr$Wp+BbXP@8%^K676$ zVnm{J@+V(rb3hft!cPXJM&?<^<$5V6f~LIeW6!hOE0=MB?ohCoQzwFt4dBRsG03MM zz$Lg6r92pxDtSj<<1U`!p(3R`5SA!O)x&uY6GWdQW7{K2gV5|RD%x)lye-NjS< z_DLQe=k^hfCFV!Jq!0fc0SnNc;Eqb4fUkv(X)Q&1^wCqk5IqIq4lF?ziaxpSafwu= zcKfo7)O3KuTqrV9CXA2zKr!PYx`=x+W*(db+WL*wQY2~}jWt`TMxzmHDd0!LNS&$= zMOC@9-e5@Sk({|9q%WldBPXANa4+U8Rtz^-zkU{WJ2$$lqGbE+KZoTE{8#Yi$v#ws z;WLP4P`^%!04K6c2R-%n^TGUGb$FG7ufqnKFh5x^eBIpwH*3kQPN;yRI?yb;fo#kR zBjJce*l;f#Pkl1okhiE+t^O`~ACpS{HcnR=&K#ylBZ@?QqP|V=2Z!@!k_+}jdxD(a zqHioy=2cd%f;v{fD@aPB`(eH0=l?HSgW7_X)ykQZ)*CYnwS~;)SpZ|{Bso)qGn}V! zcjz%o7eDWllW-|M$Nl##pYdhz7L2 z@Xrsw78Y&!;=8xO7S0!4v+ReyKKwB`yl2Tqz!7QW!QEiP$(r{EuG?O&F$O||v>(4G z=NWEs5}3GO=?QPo8F*he9ahHsWWZOy`~FV&(zX+27{EG=I7}ESby+F5`z)-=gO$F_siR%!HdSVV{&Tc?B)6&Jx$3W(VzYW5viO}>A zFZ1EQVsMPlzl-%Kndn{oNY}=EAM>L!$tWsCqN36GKZeo}OgE(K>(k-1S@sxQpkGa3 zkYL=Va?POD=p7#}g-317kEK`TTE5gFkVKwrv`+`q-~g0hv{wZmALR`*IfOh$@$nG` zmrp#2=gY3wN5xYeYZ#D4-WHQ1(mzZ*0p(OZ$si3q-b<7S_PIDf0?h+jd%@?>_YLTM zhP>%V{>p+%;Yw-5Da547q!Fz0kcmzbE4wum7a_cb4fJ zN}TNPzhS}}X@mQ6HwAoC?XPx6rR(8CVWWXpd(FznR@4LvM*Ufw2-^8C9fAk1WQfC- zw1@ug-)flQ1DAozo~V0gHEzyEJsjx1(W)OVV9S{1`8X$r0!icIoxG;LSf1q@DVX(Q}9dXfgakp$w zU!QDO+^N&Wow8k5y(fKoZAJh7j{_crUl6vD@Zno4U2>1XkR)s<-nbB&HCaQ+HYBAW zCR)3+HfyUy7S=dobJtuu)c5-pObiA*s87&Nv}i8$3=bkPo*b3Qb?ym4meDpKP&kB! zl*@}`MJF|WJu-`?m4i5y_GI`t?pRo&aLaZ2XtxCHI!U50UvdAt`%l@s>CYM}kJN1g zOlV9{@9fhgasiS?-6~VZw6R<^El0_m1+pX#Yl(kdEQ@kmpID#rxa^$;C2&^jk^*y( zS@IUdm-rH$JOl}_e$e!KAd$J>X1j7WEZUg%_>`DT98^RT=~mgWQY?HjkC?4#nWD{Z z(55m7I9K^a?yb5xExQbHSHlMzqk^f+H0{(?%Q`^`y%jQDM5UCu-3kS6TIok*3U`c; zbo6w3<)-AlILE(YKRJSROY4#bcPr28;i;W5uU0vd6#^{V88t}S9NZ*);Q%UMD;CMo zlmPr!-9d5UQ3@>ujI?Bu_t7j89xkZQEZwfs^zsFO1Pn5&q#9pKqu1c^HFR3-k+J}K%~VXca;@_-RDaI0FWR=RQtm&4wPxF| zBKpw`-RYSa;;9^dOcHK7ow%4YH)sPwfm0+d?sQ$OyldkbI)>_xY8^PvW)$;FL+n$D zU)sb<$1xX)Yq$cURuEm=6Qw{a#qsEK z4w0basy9Pq;q~-m*N$MkN;lyW5Oa2tjbgKt@&p#ni~IOkJCaqB%*Mq|o_s|vKvYS= zQ7T2c6&X3VYtg7=(M)MtEOAvUGohj-g;^3cugv-Kii;-WD6<&^WB)L?mn7uvBrC-_ zZ%K~GsdV9P5tc!+VBG?tq`-Es6VNYx=RO3U#%DbS@1prx54iXL(W#?%d3PVN5idhN zY4o2r$?iQhy6FfID77*6iR7c_)yjih;W3mMprgFQvJj{B!$-?evEUZ8sqk`c7Yp*I zpzHW?3ef?S?KfM9x9~uiytj0z*x4$H9EwabPkUy6R|5p%12>Ak|$68s@WE!|p=B4IQ8qdY(QE zz7dg2CQ8|ilDJnYZd&9xu~10uZt;d4X)WtJ*87(6K=NpFOrcB~jmp-TvKd-QDRh#Y z)nb=eh;>#Na7n~gOJWL*r5z6r zQ^_>jm?Ll_i)@q18HX#oh~#yK(YWo?c^ydZH4ZID77d_6$cEdL zq2RNh(0VD&&W+;ss+w?C)}u9P+ZC>M-u6V%a&xiGFY{4O{M8qwHCC|vV&k>j8tUD8 z`50)8?1j<$;l*Ibo$wTR!Q1YpC_pN(0*Tuohjv{H*+byY3PPf*DS4?qG1w>KrJ!~U4jlr3RnFX+WhCBkFV*eL#9H9)GDsrAWc^gaZLo z1+nuS>JwiTlE+x8xRSBnWqJ4r1R}_D@K=pJvbl4PHj$SZ*y@jIobz1#2VVwQ4f3RV z0pq1NF7IsOJ<KJ?aZ8k`tqM#omzsnPU!WO8V(+Y+iH5w zbLC4a*3UXHK-T3JAEz+t;0S2pGm8UmPK+?iX)rt?6s~771R~mBZ~2sX8M;*h;p1S9 zp42Xl8J~vk5e_JG@M$iPK6T`}M8VvIy;X2ll_Fewe2+?Coa>3fEK#+4( zYX-bt&0rPh48O}_;dh<|mn3~QrdQK-hEpY`$`}Jc0_&n3@Mzu3XB?;-yuR^cc*%t3 zgB8|BJA7{qVTR7R9*{GH{nFr~X0#gmRKjOpEg6)=!9r`ZTm1%2T5;viz4X!F4)#~I zdGnh-1(FP5EMJJJH9~)gL{2_x)I!X;1c?e-tWy<1$!hU{t{zRYjcn%dEaYUWoRz9| z$1;4OHAN2bIKh>2T80%)G{rHe@dAgC?^7!IeuqdPuus~5?UcZ07lM3=c%oZw55=5n zL1j>oWCr;wMABcQP0jXz@F#@%n}G(I5l(H8bR7{^%N1%g$x+L~qz~>qbBM8!9>6Fw zpsmrvGORUn8d0j9I?YDpSgXKdjfm$Hdp#UYG#1gb<(DO%p z6ZfJ#v8Tt6?9OJDd)osJ6($nJ#7WiA7{yUJz`c8zvV?XAiG-zFt;_o{Q@$H9s2tpB z{Pa`&z<9GFZO-(K#6?clE_^@6^^DvB{{xl|KKQD8`vP1A19o!Zv$tMCvitwTul`Q3 zf68?G5##>%@N1*PZ-I|WZ%sbKeg~2snDYmhw*7fF9>w;7Q)+`f!2heLy@# zei5EXo&$9Gzu%tfT3Jm{e+evay#UhkBjIgr!2-+PgHv;z(?siZe6nR4W$LA7p+NW$eLPPdV1UWBmfZaU-ueDqP(`1y5Gv8 zQZJuO9;R)IUzh-rJB2ENKqVBhEyLbPuYle~j&$J8kjS2$%#kqB`*&wgo?B33X^iE&35bD@(lzXwT95kN+tt?C-&lT_ z>S8AQCcv8#_<`f8j`JNs0=(Y7E7b8@^k)sZ-0*JN`{uCZ=ET-_v99*6iEv^z%KKGE zqG{8r1|%b!a`f|s?W?pqXi-zj@%F z?#e)iHYa{L{@?rukUIe&n61^B`f2=;3l{d>aEAfG(zpy3xwp5lPY1lz*oBN-u&_^u zaH%2TA{Q*|yFmwF*TO!XTdf%*fjwky>A-FJn59wYt;Nn(4_VU2E(MZ1BfPh{CGFd^ zSY#3i$lTHaZM+DTxg~A7BOY-q166h9`l%=nqR%^3fc3K7#1c~!+r##@YjRj*8C}!DTq?s5a!LC^~`|o+q``O;mJ}7@7)AZI8rN9u& z#pwI{X3!Gv0Xuyhhf4$Awvv^ue_mQ=0-8(izROe-f8oTRU5-@5nkx-+8hYH%HGscZZ=Zi?2TxN}Qrz#L8>Y^9&ZlcKdH6I^GFO z;2+p6Rs?p+E)t2+4FBL%=Cw4iCprhs&1&f|Sd-4LQ%`gC>~;%*gU>1^9QjQB+*POC ztZRTvPeNkx*|%HlS|3=t(i=0jv(lY3Cc52Va~Nkq>4ztReoy)Y^PzhuTQ<diaUF*<-8*ryhrer!5P#E>V~r&eHun#x5n;--4lBpX2wps zW?E_ffY2*LqeJT$yc-o``~&zRX2AWTd)j^2J>#BrUve+B0pbR6TjDOny@>}Ak02gL zq!5|J2qK>-Au5SFViYl!xX?D=w&L+iAw$_CUqMbFE$o2j@1Fm7Ui3_R4!1u~{Q>_5 z{|ldir{Orf1aHC_7;Jy7{c!tC`)vEA_J#Il$5#4Ir^(!oxhr!Y=E2OPm_(+BOLXb% z$9T8-K=?@bvEi!lD*hy&_9|SCyTATRf?EQ_d%%0Vn`yW4HU6H4PYK6`mxQa`XSy$T zFLbv$cn1hcAuDtWWuYdFG;V0z-gs~0qmAzny&!s7bW0wt;>m8{aK^eT1o2ziqHB>5Babs-cwMNi36<|#dt1C(Qw)08WeJCq^H zYbv0!s5GjT>ZZ0(`>0dYUDSW6J=Dw8`_w1YR}c;&Lv)A(i68}}flQDC@{6q>5BNW~QxO@EfcCdk08)owMe`?v@jM8eeR@RkxO3+&dM)F_`YTqf683B7 z1VI`^rlqnn!|p_qn4caDj#Tfwe)q%V0QItoBR$ zeZb92RR9UV0T)0JK6>G;&d;v#n%2nc*Z-vjbc{&49&s+0l;-D`mec`#aO{2O770)#B)h4Y76C$3aEa+yPHfwFfcZqN%3yXVE!3rkk z_X~%;oaL!vPfw>!6<*!OIyg=@`nR=E4nITJ`?k#?o~~+>nIg8)pzD!v0GrsARPhDc z>rc;TvO(Q3+kLpq$s2yS?-Wk(w(xy!nkJ;jpko)QH&}Fx6-N15Ez2C0CZ9-0vkbH5 z&bIe+hgz3?A7cWZE8}>}l1oi2WE*!jp{EmL1%r(@&LM^##4ZIxAiGVtmz4T9bqb-4H==LUHLZg7zMI1nQa67TSp5Z=H5w$=Z{ zWvBh7jcj3OvvhINWICtqnmQR{PqnBjmGk_`C$dCex-0lrWTc9Dd*deA3G7M&{RE^3 zU+aABAgZ^m07n2NzH!L+iLcNP_s#=f7rywXsmY-cIr`q+!wdX#PS~jDdgzH?K}vm>Rstbt|dC?|CEoQ*?gYecCIfH(am^* zB18^u`KKH)>=K1+%pmFCP6qVO=|IL=j$(B zZm7{G(fQat9}Xva?E!1nBaB{lNdbDL}J>7X8OKJIB<)9q2jjZ}sQKYQF zX%&P!aG6XHvI330xfT`^V~={xl2*!-R#k_)+-H(DCDvt$ij`&-B2Po&A(8>22M_^= z1Lg6gd<95+W<8hYK@2nQ}#S^_d$fiEUzNZj<2AdSi{`~ZaY+TsU-K5Z=4ma5f~5QhtN zyLXxbM6SPX^c11`hFuzN!?55l9O35zB{wolTy73BB50}iMrYQ~>4{_E%i}|bRY;~w z(!=Ofd2tEwFp0jLcuLQMZxIr%=h~IV0zZ`Eo9q9OYAsCzfl!RyRIt@e9!*!&9j=E4 zZ6gxTHR~!maYD7N7xho;v9KE?w0O(f6@lV;9i^_GOhs2*Hz~HOisQr#JIai)Bqxv- zp|%j}_Bf}8&;oz5{saW34(%MNoO1r8K+x$Z_KOXaP<(^9iT>-^LgDMpI-OJUn;Fzx7LMu@18JnRh)|3n0E_kF+ zuAQNlJp0Ux7&Aos!ovKlOHE2t9tmPI-eM`{(pe`S)lVKeUo7M(DxF+~{>VT2jZfT!j#|T@^6#lpmoGJRa{!f z7aO6F37OapjAf+*jORpjOHcpRqiGOvHA_zo)4HLzE!phxr|jcjW2=OSD)oMGirN}XYz=Ng`oy^0>+-`DYYLp$+Y}C}|On8}u++pMpZMF;7OcKy8 zr3^>*;He+GwTPhE1*n+0XUb9S6Z?THCCv79Y zrf1%MTN%Vz#%>iqfME*oK8I`4S?QF92q=FvqlNLEtP{z$N|MwgP@AkotRZv5VdAFW zj3Gabqq#k=DNka<%@<#NQ(@r(AdU_YfN|BpobSb!e(YoCK}!NC!T>8&-Jdg$UnT=( znOI*~&=y|yWz2)s<$2j<+7t@6(uTKsHo?9RhunCKm}q>HWig5$d$_g9&g9GT9n-+K z6F~6om`{FJe%G#*wJeEP?p>5h0?)`rWt|}IDg-eG%MA?qT}7~>#t7Ng($ln$tyIWI zB4UIO>JpJ8mMOF}ZT@iRr(>o^GKwy&)}#AvgmQNT(UZa+P^?Bw?+07m5L8bKuLz7M zZ$xDhYxx|nTqD_!W->9B+gum(wAfT0r)J>&$!vJDv1oK9s)A!ND9FSM6(dOb#G*

3bNFmBv9b>LvJJic+O4O z12)h1$g&{HoQm-Lpe6Q`N*zkl%xu+qz&w#S*)60wBs<`i%xH~t8^$RSy-~ag*z<*; zBjt+MSO0HV7YpJZ}*4|i7 zW@1eg9r>GzH(dCX(NtPDMEXQC+hXnPtu9%HI&=ZUxRY+ls8PcJAdfO;kPZ@Ni35*S zd$XFOsR(MvVb^5v%@6?ylF76{27J+=ujA``0fMP!8B!q7E+Ee@KcO->=T~snCAF7p zL0w~osRGI+j++)EzTw9~O?dY3URaGLH2G8_#mfSq2>BgH(-jWn^{}&8>`2=nWL~l; zi#B3ubJjzaR~Pa`3pkUlk|+)`}6Bp6N1Z3VrRzGlU;Q?bjG@?C@@lT*Ik0Eb)!_lyYvr-v0pEBM6tW_wh&3(;+}=9>t+ky86G9~AKr*@ zxxCtHt*&TLE2hu?!qn5ANw=BNgNI({8Hyrek-xCt{q5rw6;WKx-nKbJ0zp5BCEES) z-nA2sq9RtI&^1I)1;MVzmm~cNo9P<$*_@5;fmmucmR@RmSpx#oz`S(%!71ZcxAP@6 z(;mMAu44c;g##EC2!@Zfg}6`KYo^0Vs4Zf*^}E6xq#caNli}IP%Y zZ_A)>fd6^?mLsJ|%n5ga&wdXio%}NT`U&wfhFz}0 zS{p4ItHUa)P`L5uZlkE{;wsuXj&r-FM&(j;h!wJ6H9{NM52C;WKWzBrQShB{H?O_6 z(-vEE{~TDdF6%a0spNSKY1)azD4+*g3oPNq#OA~yE43`0<3yE)cef_2 z%saY*0Fl(P?vh33RELeKXREA}0Tu4e@@eO74>cR1Uk52Wu?A}wsSD96vlxrq!Vi3TtyG-T9axzu^;u-IZjehPDwlIn zLJcen;Imz;w8d$~B(ii022#_Mhe#v>H}7{tlPpoR@7ig$&7{AYkk6`D3=$=1m2J&5-c6;O8yldN z-k!~!R5sVNeR{Fj-zpdwOs-6&j60Um@)3-pgGuAu44s{;e#WeDegk9o zd+EVwE zp>*(fXU8_F)`Bb`_OUt0+#}D>A>bS^!>9z>Zyv$K@UFAq6_)(ScMdwbIZt~a_MV59 zPwi9dp^2*YI(PHHxqdU=kh|n=ySB^Sh&fJkr@S=?fy6OgPh#Du>nhEaPVdFAT(ScD zNnT#K#VXUbZa5nu_mJD=I#0Tgijuo4phuHOZEfoA8PMqRDOHpTKBgKxMM-1#A7GAF zN@Y8p$r%--$P^tj7#p+&(Eb@*e&_y#fVz&k z7h3Nq-M=kR4U-3QIdu07bfvkc9`clmk+Wc7@zv@vsjjR=6wTI1+X3==$L>G7Jj(J0d4p|l)V6u!i#(arX(DHs zAew+g2)IC}<8+?S*NuY*K8Xv=3?L4(BXYq}VW5gxQfh*^;V!0=sN{Kp->%u1MVO&- zbN&dke9naDbBQ~bic4FL)NZ|@8{&bGmurl1i>rI)f+>LI)5ss3I@=3zm_r6VAfO!U zLgX@;Y%;mI$YC@xiH8C~P!=c4NFoXUxOS`A00kS5Y+ISSLrj&_BSR9I?7GhEi4=_A z!Z@t(Wzd?byrCI(aCNG3^?;2sOfoM^Y$M(@HWW2xa!OaE=K~^fp;*$5Lb1DC^c?#L z<;HIvkB5g6Tf%2u;adL7q{V2L9U(ng2~viZ@POYBVfQLbhjjJGr}d-GdeNmEaIPGHw>$*+hO0TwEyF3 zbx$YmXEZTHCIYn}F_07ug9T1s@18}58W}R->`2V%{~p5^%w4?rO?*V-eanhT!*=5) z5??6JLBLOEO{Uw9=X11Mtk%|4C!AbvDLQ2xQJJ$L+tJ;do7*RXz^pz_KAF)Thos6o zXU2;tUS&1o*iP9R8*vq;xSSY)!CUOwN;#!%nz26v*uI35&bDdtBEN}1uQxZM z978E}tZZ)}nqJfng;wj!)753)grpM)=EwJXT4b9iwSd8)GY0k~7-!o+DAivCH}qDX zUeU-VuQS$3Y(0tRH7?{fug&GC$npeJvfr+qrliIbVikms$QT@VRT9@W5Pf^Aa?+8D z43?oe5D&*V$f_B-#M5R?!y65*#wLF&6Snnmqcb#ciV)RGGP6<_#ImN3aQ7;Uj9nA| zm~M|GvFv*x{O#}E%oP;%b^z8b9FWOO8Fax815RMVNI3@WibCY} zLGA(Z6%OhW8JkH*X(~?xDxMx=oW1xUp$ok}oNuPxb#RbnY{^YKr5a^Vwm$ z7;Px+B}EZX_ChLkyD@EW^wy8V%q@7nu1NEwOlo4m;BFH?O-%l0jiu-7$}7$;GG(3T zX%5V0BRFt1_ZbsdTDlXObA4_Y?`qL%`Hno;8n9!*l9CclQnd2$9_Z-e7BNhC5@w@Z z^1e7&B38~PmcMyh?}8a(z}S&XA*v_y+Mpd z+zUM11F~A9Srl+R8HVEa?qU7b2bnk$Da2=d{;Cxza z>}B(FwWC4{2wjvDsKyqyoPoi06VnJwT@@FA)CXG#ASMrub)z6XpJtyMlLG=p zk6fqnLZOwTNXf6<3a&LrG#v&Q zjG06sdb{Ch!5KjmPK;v?Fpi+1tmM%ec8$gmh?%?d)oae$oHuvhX3QYP@QGRL69ZD@ zSZ^>37>~6!AnDjQG3n)-PB8ZsXWyszG;z7W9i5*frtcr{k@7QmePx{++_==hAP5RQyJ#xLZ*?!%yj(5L=3haK^z{Wur1xMeRSZ6m5o6GiAUGaFyk%Vs+;EX+q~~dIQMh6oCGY?_=`~)+DCH` z5SdStuVbpRoEkS*$GzhFJ9y+mG|{?jfalx@GW)%Isb8pfpZgto`jNUliNghd>L~jj z`)e$2u}H^yB-&tYZZt`f$&bWXWL?etSg|U4RXUkW_ljy>j8UzP7>hf4!fHO7!BW_P zlw(DWz+BX3tZDs28D41!t(dSaO$n^x5F72_<8(H13JNIhOh(gbbeCS_5T!rM?^)_@ zrLGRznX~O&XGf{4o0L3=SfhT81xO{?AgkG7)_i7Vn(~VNt_Uqg)N)eJbPCI4^YxBI z--NkQARKw58^+;J6bQVjcAH3exC~S+)YQ{FAqnE|Fyr~>e4Ih4J+9GS7tD|G;_Ck9 zjywQ)c^|3%`O+xDhq<*}emUr+o%8tRQY}P(3{fyPSQt|-Yst$ghdfP8Dm&3046z#( z=-r>cq-|wp0AlOy!tRz~yZS2d{zM|J+`FSh$M_WvhE`w!G;>2yNDFVkYiqk^pUup1=jWptTGE~N z17KcH;aS^gvt_JAB}gmK2yLNK6OvWncC@C@uCzFt8%lPhlQQM(QT=?nW(bGh#(fSX z1UcW83yu)Hdp^Mh3l@uqHIPf>gUDR!aEU_vOEiSEJux=V*tt|kQQ=`*Y8FyKA{mx( zGT|_5l~z)=gak!VA(9wrH4dq7S|lHlA$z` zMs_cxa+Vl#TW^$Nv0yeczEJIm9u7;AvL30^n2*%f65VSZBv`g) z*N4vib-Q#I`)EJW)Wme>1nqeS@RU=r%q4{up$d83B@g+lKp;$}u{2qV&(ozOTOX%# z-b^Gw*K&(Mm|L2#oFAc;2Dzr0r;4f38A*`9h%C#inSbX<(hU_veE)-_n+gW~I}cr# zt+ZgNXUBD5PrN=VZZSrPwhm+1ZtKNFpSB4r-Y=z6Q4a4`uc7mFN%-N{<7|a{M zkyR$92M3xWf;fWviNH9j-oC3WuB^Z^LM6Er)GT2QJp+vttHGGSdZSo`qXcdO=G)>7 zh~*B>SjaKP5$Y;3mPVOYVM-k+kQtm|a=G$GKh|~sI_N0cMu;$sp|tuuUtL%#E>);{ z^nbWY?M1O2QNy_1~#EBW@zaLo2DE>6t=@SXFBN`4{ z5{Yvq8yo&C)DxWU%77=Z&AN3VL(1cfUUwo3hN1pRCn#JYg4{+0t)iFG6}rRXXp93o zcH$Tj*yz}LR2AbqdL21jlTwmHli5?pJi!=@KpIKH(7c-6+Rm2x`XFK{spVksi(4Ow z#!a|mpi#4>0h(!oMGZt>HMZ80$1(||Cflrciy=7Mi|Kk$(dvv<+8MW|0SfTCLrh?jK_(rQ;#v-$=8C$C(O9Qelx|zM8-effp;`Z5y=l$>U-}HD^!fRehM9v+R?^P~!mZw7BrM0@gIFzFC-N8MrWHm*DM*rm$s}uj{hodIdzX)GlpTW^%zDBAH`xuYnZ`8HZqu3P3#W14`E3jJ)NsMJr z2<_?3V?+blMEKCmj3|9c`NLidqQFo4`Igwa+JI7$iK-Uo^vO&!F+^$&b8vW)QNU(s zK7aIg%<`W!l%KZZ)WqlM3Q4Id=1wcWAkK{WHequ}!Lt3DpWeT;WIjC{ZEcN4!pG#w z_~}G@TCvmGRsp-c=z`|~(ld4s=M0Qb=E@USs@Q@!5Mz!RsYlodxd?H9Cg+t_Drf0x zNhYzZTBC^pHs!it!P4Jx&SBnfvLFp3)&hO!czChPiDz>cjFDH|&`Y2#6IoSnA7hpI z;0p9cQA$>hV4xD#o`nh3$(YlTWexe4W2oc^F69O3$4SUHBZ0yhq8~E!Vd)(0M*e90 zfWJ<^*O&Og(U{63uz>rEy=e2A{*GP{!cI0*L4Gff1Xl3d2qpjX`EGkMp}$)Q+5g(y zLO(JS)GwGBgXEsp|996Jhnn5Pv#O}gG}72$AqzxCo<4$<>_Nh3Lq#+y0LrjuP>|P0 z(aXa_`Mt}<^SwpAvwncS554nxPrA960I3*quk(EjUVzVt;tTrq*Th4{B<#c6D3=58 zeK}gH+#ks$9P6Mjbooxw$Pc4gkbt_AXgGST{4XOSx$g_XVNB=wld-YUF_$O@ArZ4$ z7{p|=r|v;jMhL3!`1OK&C{4%vaU6+;i!3a{rc6ee&hi^jx90I2_k;f%X#6w6=l|2t znHbmD)|rivj6%3XRwX!}Dxpb;E=RAT5oyJdPzeMNDhF;pRpZ{m zkOxS)Gu?nYH+ue$aqb;LIEd#k^{Dp&S)Rc1A@jJUWX$UkvqnxGCp)4NDaXfx!7PZLmJ!%wh zFIk-wxdxM}=XLMi@EOa2<%fGtPvNBmwRCjx#|t1Nc~w)BJ2Tf8!#?Uf$Z&FX#Q=kw6ty3jo(>ZFuIv*vvs$L`fUeL z?&{`<)g2pVX(U`E9zz{Trw$hN>U$#;)`ohb7g1@Fm01I!p*?NMRAsMM9*~1dV%WVg znG|(N*a&kz+erw$avVY%hp~S=yFJgmOw%2v$rz8gT~Q>>0R_#S3h1VhRKyF5fHa5_ zvMAyfc%UY9EX!qwmPTP+{le(1Q+P&)ohGB4O_oJ91iA)=!V&tA+2-g~i%bVe;0-ln zUg`+VB28E^R*Xc3yU$>Dv1(~uW0t-q&Zwd=8=wmGja;3VnitqZP=R5kM#JU|j45H| z8G15XqzrN?4F%Iuql3*J-x6(tU9z?`jUUk=2#s=%XW;xBqI^MzJ(c8e=xVSRCY0F! zc-Uv;Z5D5xuWZmp9geF7M$Xq`&=uD+WTx?TmgXrviuQAlb#9CJDV>cCI**i_V&@2S zGdPIQdhC!x2DOYy;%=riQPU!`B^1}!qu?dy&Q%3GOW@WAs_;{6!Ky*@Xi58$=HWHubuTmQUs)ov>ksPGu5dcGu3rI^YPQz z8!^kiUE@tOwXbcP4%FWgGty)EKvdnGO&{P75?ae~xGQ_~zt9-;UR!>bAZJ0Pl|YRzLY);V=ndKoA%J2w2FqVDn1UPogp2E|O`}*jXL3WTjdx zH%uOrVq>nauAvuZT;r+9#q=mPWDAqnHPVUqo|%@?OGTTafLg}${&jL_=&WF#g(^QN z?-XEA%LT)jaI6trAd~m1!#EqKfL^huqrOP16vrKbWKiS)71HveF>+GFX_#bt-c(L}}Ai5l>eHIxQ5`#^yfI?o*Mv6ZQkN6HLC zrPNLx7PH}Q{d)D-4$oq@@a5j_2LsU%X4fDW^7*OCWr-JS0D5-s$@ zdvorJQhb8J^us>UZI^Ro&42T_5<(d4Tz%7L_I22I~+*OvN*%K=DfIn&O?U zK1G(ya>XYa^A+EOOp0Ij3Kai*`6~f25DWSFo`5NJMrlx@OiU1^wWro#`+Y>KQ6vVc zF}d%v$1!Wtph&Z#CiokZhZ^qf`-yB@uT=>{-+maRLFuhVsa2EvLzY?L^%Pw32J!h* zRjOdnG}K2ZlS}Csap9xpQ%xEv_=&n}w~SG_7e-OkWXTdkG%nJ2nc*v4k%#dS$~nrQ z)v_{Tl@w4E{c<6GV<6E)P?u}AbC-rrts>Q=J4HD>voe63hKy0jlhP*Tb0#l?Oy)RF zj1nta#(BcYZ7KXS#G{Cea2avdw>??8cDhI}5r`C}jVunxOLNDC{JpFDDsoHTq>5U( z>q&+TvNGd50kSbCvbhMWr1I!A<*&!YOQuN|8vHPGjG#4=POd!pHahN@O^TIh zQ|hELeNKeI+4Tcd%UzUjyjg{ITU?R9Y6ed|_B_9l8uZxakUvHH1tt+x|1f-e_6kKBV$C$IW}0CZ0gC8|fqq#HL*7>Y z*W$#P3$|ZzxZ-ld^6D#hC4y>*nZp%k4si38(NIrC}mOQi&2RMyxpT5+q8JEJdm` z=`yHg%92fExEv#Bjih^j{`X}CpaKYxRGH;eEJV-kl!x3^OoW?qWV&sI6u3{wEAM(4 zv*-Kv^j_cVl?+x(!%hB$aK}0wmcK7NT=N5d4L>0e9C~H>2eIRqrS;=;{&W55+s@%k zUq=qVJO8p?a>-6^@!)j$nx^>&a^co(cxZM9ucvZSxb_GHKOqntdc}da90-Jd%fTxS z1P9(OfxkNj)D7s0L%(&MHl;Zy$(FUtCXlath4skH++nZUUU}q?VmVIT<6~iHHa7Ho-qnXqeoXJdSD`YB` zXXGVJ`m;nh%Cg~>BpXZ>x-S*$!PzSl4g~u{knk6#>C*UZ$RRlJAAUx+h%VhLtY(null); - const [throughputData, setThroughputData] = useState< - NewOrderThroughtputChartData[] - >([]); + const isReady = useAppSelector(state => state.globalState.isReady); + const dispatch = useAppDispatch(); useEffect(() => { - conn?.db.state.onInsert((_, state) => { - setState(state); - setThroughputData([]); - }); - conn?.db.state.onUpdate((_, old, state) => { - setState(state); - - setThroughputData(prevData => { - const next = [ - ...prevData, - { - transactionCount: Number(state.orderCount - old.orderCount), - timestamp: new Date(Number(state.measurementTimeMs)), - }, - ]; + console.log('App useEffect - setting up subscriptions'); + if (!conn) return; - return next; - }); + conn.db.state.onInsert((_, state) => { + console.log('State inserted - dispatching insertState', state); + dispatch( + insertState({ + warehouseCount: Number(state.warehouseCount), + measureStartMs: Number(state.measureStartMs), + measureEndMs: Number(state.measureEndMs), + runStartMs: Number(state.runStartMs), + runEndMs: Number(state.runEndMs), + }) + ); + }); + conn.db.state.onDelete(() => { + console.log('State deleted - dispatching deleteState'); + dispatch(deleteState()); }); - conn?.db.state.onDelete(() => { - setState(null); - setThroughputData([]); + conn.db.txn_bucket.onInsert((_, bucket) => { + dispatch( + upsertTxnBucket({ + bucketStartMs: Number(bucket.bucketStartMs), + count: Number(bucket.count), + }) + ); + }); + conn.db.txn_bucket.onUpdate((_, _oldBucket, bucket) => { + dispatch( + upsertTxnBucket({ + bucketStartMs: Number(bucket.bucketStartMs), + count: Number(bucket.count), + }) + ); + }); + conn.db.txn_bucket.onDelete((_, bucket) => { + dispatch( + removeTxnBucket({ + bucketStartMs: Number(bucket.bucketStartMs), + }) + ); }); const subscription = conn - ?.subscriptionBuilder() + .subscriptionBuilder() .onError(err => console.error('Subscription error:', err)) - .subscribe('SELECT * FROM state'); + .subscribe(['SELECT * FROM state', 'SELECT * FROM txn_bucket']); return () => { - subscription?.unsubscribe(); + subscription.unsubscribe(); }; - }, [conn]); + }, [conn, dispatch]); - if (!state) { - return

Waiting for data...
; + if (!isReady) { + return
Waiting for data...
; } - const measureStartDate = new Date(Number(state.measureStartMs)); - const measureEndDate = new Date(Number(state.measureEndMs)); - - // If the is in progress we calculate the ellapsed time based on the current time, - // otherwise we calculate it based on the measure end date - const ellapsedTimeSec = - Date.now() > measureEndDate.getTime() - ? (measureEndDate.getTime() - measureStartDate.getTime()) / 1000 - : (Date.now() - measureStartDate.getTime()) / 1000; - const tpmC = (Number(state.orderCount) / ellapsedTimeSec) * 60; - return ( - <> -

measureStartMs: {measureStartDate.toLocaleTimeString()}

-

measureEndMs: {measureEndDate.toLocaleTimeString()}

-

total transactions: {state.orderCount}

-

MQTh: {Math.trunc(tpmC)} tpmC

- - - +
+ + +
); } diff --git a/tools/tpcc-dashboard/src/ConnectedGuard.tsx b/tools/tpcc-dashboard/src/ConnectedGuard.tsx index daf75de0e75..bca511e10f3 100644 --- a/tools/tpcc-dashboard/src/ConnectedGuard.tsx +++ b/tools/tpcc-dashboard/src/ConnectedGuard.tsx @@ -4,14 +4,14 @@ import { SpacetimeDBContext } from './context'; export function ConnectedGuard({ children }: { children: React.ReactNode }) { const [conn, setConn] = useState(null); - useEffect(() => { if (conn) { return; } DbConnection.builder() - .withUri('http://127.0.0.1:3000') + .withUri('https://tpc-c-benchmark.spacetimedb.com') + .withUri('http://localhost:3000') .withDatabaseName('tpcc-metrics') .onConnect(conn => { console.log('Connected to SpacetimeDB'); @@ -21,7 +21,7 @@ export function ConnectedGuard({ children }: { children: React.ReactNode }) { }, [conn]); if (!conn || !conn.isActive) { - return
Connecting to SpacetimeDB...
; + return
Connecting to SpacetimeDB...
; } return ( diff --git a/tools/tpcc-dashboard/src/Icons.tsx b/tools/tpcc-dashboard/src/Icons.tsx new file mode 100644 index 00000000000..5b4af5ca696 --- /dev/null +++ b/tools/tpcc-dashboard/src/Icons.tsx @@ -0,0 +1,143 @@ +export const ClockIcon = () => ( + + + + +); + +export const SchemaIcon = () => ( + + + +); + +export const UploadIcon = () => ( + + + + +); + +export const PercentIcon = () => ( + + + + + +); + +export const RefreshIcon = () => ( + + + + + + + + + + + +); + +export const DataIcon = () => ( + + + +); + +export const ConnectIcon = () => ( + + + +); diff --git a/tools/tpcc-dashboard/src/LatencyDistributionChart.tsx b/tools/tpcc-dashboard/src/LatencyDistributionChart.tsx new file mode 100644 index 00000000000..772dd9cbf79 --- /dev/null +++ b/tools/tpcc-dashboard/src/LatencyDistributionChart.tsx @@ -0,0 +1,53 @@ +import { + CartesianGrid, + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; +import { useAppSelector } from './hooks'; +import { useMemo } from 'react'; + +export default function LatencyDistributionChart() { + const latencyData = useAppSelector(state => state.globalState.bucketCounts); + + const chartData = useMemo(() => { + const sortedLatencies = Object.keys(latencyData) + .map(key => parseInt(key)) + .sort((a, b) => a - b); + + return sortedLatencies.map(latency => ({ + latency, + count: latencyData[latency], + })); + }, [latencyData]); + + return ( +
+ + + + + + + + + +
+ ); +} diff --git a/tools/tpcc-dashboard/src/NewOrderThroughputChart.css b/tools/tpcc-dashboard/src/NewOrderThroughputChart.css new file mode 100644 index 00000000000..f54c4025a95 --- /dev/null +++ b/tools/tpcc-dashboard/src/NewOrderThroughputChart.css @@ -0,0 +1,22 @@ +.chart { + padding: 64px 32px 32px 32px; + border-radius: 0 0 16px 16px; + border: 1px solid var(--shade-3); + background: var(--shade-6); + + --text-color: #f4f6fc; +} + +.reference-line-label { + display: flex; + padding: var(--Spacer-1, 4px) var(--Spacer-4, 16px); + flex-direction: column; + align-items: flex-start; + gap: var(--Spacer-2, 8px); + color: var(--color-green); + text-align: center; + fill: var(--color-green); + border-radius: 100px; + border: 1px solid var(--primary-colors-green-100, #4cf490); + background: var(--Shades-Ash, #141416); +} diff --git a/tools/tpcc-dashboard/src/NewOrderThroughtputChart.tsx b/tools/tpcc-dashboard/src/NewOrderThroughtputChart.tsx index 3842f586536..33f6bca73d7 100644 --- a/tools/tpcc-dashboard/src/NewOrderThroughtputChart.tsx +++ b/tools/tpcc-dashboard/src/NewOrderThroughtputChart.tsx @@ -9,19 +9,8 @@ import { XAxis, YAxis, } from 'recharts'; - -export interface NewOrderThroughtputChartData { - transactionCount: number; - timestamp: Date; -} - -interface Props { - data: NewOrderThroughtputChartData[]; - runStartMs: number; - runEndMs: number; - measurementStartMs: number; - measurementEndMs: number; -} +import { useAppSelector } from './hooks'; +import './NewOrderThroughputChart.css'; interface ThroughputBucketPoint { elapsedSec: number; @@ -31,7 +20,7 @@ interface ThroughputBucketPoint { } function buildTpccThroughputSeries( - samples: NewOrderThroughtputChartData[], + bucketCounts: Record, runStartMs: number, runEndMs: number, bucketSizeMs: number @@ -43,19 +32,9 @@ function buildTpccThroughputSeries( const buckets = Array.from({ length: bucketCount }, (_, i) => ({ bucketStartMs: runStartMs + i * bucketSizeMs, bucketEndMs: Math.min(runStartMs + (i + 1) * bucketSizeMs, runEndMs), - count: 0, + count: bucketCounts[runStartMs + i * bucketSizeMs] ?? 0, })); - for (const sample of samples) { - const ts = sample.timestamp.getTime(); - if (ts < runStartMs || ts >= runEndMs) continue; - - const index = Math.floor((ts - runStartMs) / bucketSizeMs); - if (index >= 0 && index < buckets.length) { - buckets[index].count += sample.transactionCount; - } - } - return buckets.map(bucket => ({ elapsedSec: (bucket.bucketStartMs - runStartMs) / 1000, tpmC: bucket.count * (60_000 / bucketSizeMs), @@ -64,59 +43,95 @@ function buildTpccThroughputSeries( })); } -export default function NewOrderThroughtputChart({ - data, - runStartMs, - runEndMs, - measurementStartMs, - measurementEndMs, -}: Props) { +export default function NewOrderThroughtputChart() { + const runStartMs = useAppSelector(state => state.globalState.runStartMs); + const runEndMs = useAppSelector(state => state.globalState.runEndMs); + const measurementStartMs = useAppSelector( + state => state.globalState.measureStartMs + ); + const measurementEndMs = useAppSelector( + state => state.globalState.measureEndMs + ); + const bucketCounts = useAppSelector(state => state.globalState.bucketCounts); + const chartData = useMemo(() => { - // const totalDurationMs = runEndMs - runStartMs; - const bucketSizeMs = 10_000; + const bucketSizeMs = 1_000; - return buildTpccThroughputSeries(data, runStartMs, runEndMs, bucketSizeMs); - }, [data, runStartMs, runEndMs]); + return buildTpccThroughputSeries( + bucketCounts, + runStartMs, + runEndMs, + bucketSizeMs + ); + }, [bucketCounts, runStartMs, runEndMs]); return ( - - - - `${Math.round(value)}s`} - label={{ - value: 'Elapsed time', - position: 'insideBottom', - offset: -5, - }} - /> - `${Math.round(value)}`} - label={{ value: 'tpmC', angle: -90, position: 'insideLeft' }} - domain={[0, 'dataMax']} - /> - `Elapsed: ${value.toFixed(0)}s`} - formatter={value => [ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - `${(value as any).toFixed(0)} tpmC`, - 'Throughput', - ]} - /> - - +
+ + + + `${Math.round(value)}s`} + label={{ + value: 'Elapsed time', + position: 'insideBottom', + offset: -5, + fill: 'var(--text-color)', + }} + stroke="var(--text-color)" + /> + `${Math.round(value)}`} + label={{ + value: 'tpmC', + angle: -90, + position: 'insideLeft', + fill: 'var(--text-color)', + }} + domain={[0, 'dataMax']} + stroke="var(--text-color)" + /> + `Elapsed: ${value.toFixed(0)}s`} + formatter={value => [ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + `${(value as any).toFixed(0)} tpmC`, + 'Throughput', + ]} + /> + + - - - + + + +
); } diff --git a/tools/tpcc-dashboard/src/StatsCards.css b/tools/tpcc-dashboard/src/StatsCards.css new file mode 100644 index 00000000000..61269d276a5 --- /dev/null +++ b/tools/tpcc-dashboard/src/StatsCards.css @@ -0,0 +1,23 @@ +.cards { + display: flex; +} + +.card { + display: flex; + padding: 24px; + flex-direction: column; + align-items: flex-start; + gap: 32px; + flex: 1 0 0; + align-self: stretch; + border: 1px solid var(--shade-3); + background: var(--shade-6); +} + +.card:first-child { + border-radius: 16px 0 0 0; +} + +.card:last-child { + border-radius: 0 16px 0 0; +} diff --git a/tools/tpcc-dashboard/src/StatsCards.tsx b/tools/tpcc-dashboard/src/StatsCards.tsx new file mode 100644 index 00000000000..37b67921c0d --- /dev/null +++ b/tools/tpcc-dashboard/src/StatsCards.tsx @@ -0,0 +1,124 @@ +import { useAppSelector } from './hooks'; +import { + ClockIcon, + ConnectIcon, + DataIcon, + PercentIcon, + RefreshIcon, + SchemaIcon, + UploadIcon, +} from './Icons'; +import './StatsCards.css'; + +function getTpmC( + measureStartMs: number, + bucketCounts: Record +): number { + const measuredBucketStarts = Object.keys(bucketCounts) + .map(Number) + .filter(bucketStartMs => bucketStartMs >= measureStartMs) + .sort((a, b) => a - b); + + if (measuredBucketStarts.length === 0) { + return 0; + } + + const firstBucketStartMs = measuredBucketStarts[0]; + const latestBucketStartMs = + measuredBucketStarts[measuredBucketStarts.length - 1]; + const totalMeasuredTransactions = measuredBucketStarts.reduce( + (sum, bucketStartMs) => sum + (bucketCounts[bucketStartMs] ?? 0), + 0 + ); + const elapsedTimeSec = + (latestBucketStartMs + 1_000 - firstBucketStartMs) / 1000; + + if (elapsedTimeSec <= 0) { + return 0; + } + + return Math.trunc((totalMeasuredTransactions / elapsedTimeSec) * 60); +} + +function StatCard({ + icon, + label, + value, + unit, +}: { + icon: React.ReactNode; + label: string; + value: string | number; + unit?: string; +}) { + return ( +
+ {icon} +

{label}

+
+

{value}

+ {unit &&

{unit}

} +
+
+ ); +} + +export default function StatsCards() { + const warehouses = useAppSelector(state => state.globalState.warehouses); + const measureStartMs = useAppSelector( + state => state.globalState.measureStartMs + ); + const measureEndMs = useAppSelector(state => state.globalState.measureEndMs); + const totalTransactionCount = useAppSelector( + state => state.globalState.totalTransactionCount + ); + const measuredTransactionCount = useAppSelector( + state => state.globalState.measuredTransactionCount + ); + const bucketCounts = useAppSelector(state => state.globalState.bucketCounts); + + const tpmC = getTpmC(measureStartMs, bucketCounts); + const theoreticalMaxThroughput = warehouses * 12.86; + + return ( +
+ } + label="Measured Duration" + value={((measureEndMs - measureStartMs) / 1000 / 60).toFixed(2)} + unit="minutes" + /> + } label="Warehouses" value={warehouses} /> + } + label="Max. Theorical Throughput" + value={theoreticalMaxThroughput} + unit="tpmC" + /> + } + label="% Max. Theorical Throughput" + value={ + theoreticalMaxThroughput <= 0 + ? 'N/A' + : ((tpmC / theoreticalMaxThroughput) * 100).toFixed(2) + '%' + } + /> + } + label="Total Transactions" + value={totalTransactionCount} + /> + } + label="Measured Transactions" + value={measuredTransactionCount} + /> + } + label="MQTh" + value={tpmC + ' tpmC'} + /> +
+ ); +} diff --git a/tools/tpcc-dashboard/src/context.ts b/tools/tpcc-dashboard/src/context.ts index e136f2b9a8f..c98bcc22443 100644 --- a/tools/tpcc-dashboard/src/context.ts +++ b/tools/tpcc-dashboard/src/context.ts @@ -1,4 +1,6 @@ import { createContext } from 'react'; import type { DbConnection } from './module_bindings'; -export const SpacetimeDBContext = createContext(null); +export const SpacetimeDBContext = createContext( + null as unknown as DbConnection +); diff --git a/tools/tpcc-dashboard/src/features/globalState.ts b/tools/tpcc-dashboard/src/features/globalState.ts new file mode 100644 index 00000000000..a18efe22581 --- /dev/null +++ b/tools/tpcc-dashboard/src/features/globalState.ts @@ -0,0 +1,97 @@ +import { createSlice, type PayloadAction } from '@reduxjs/toolkit'; + +function createInitialState(): GlobalState { + return { + isReady: false, + warehouses: 0, + measureStartMs: 0, + measureEndMs: 0, + runStartMs: 0, + runEndMs: 0, + totalTransactionCount: 0, + measuredTransactionCount: 0, + bucketCounts: {}, + }; +} + +function recalculateCounts(state: GlobalState) { + let totalTransactionCount = 0; + let measuredTransactionCount = 0; + + for (const [bucketStartMs, count] of Object.entries(state.bucketCounts)) { + const bucketCount = Number(count); + totalTransactionCount += bucketCount; + + if (Number(bucketStartMs) >= state.measureStartMs) { + measuredTransactionCount += bucketCount; + } + } + + state.totalTransactionCount = totalTransactionCount; + state.measuredTransactionCount = measuredTransactionCount; +} + +export interface GlobalState { + isReady: boolean; + warehouses: number; + measureStartMs: number; + measureEndMs: number; + runStartMs: number; + runEndMs: number; + totalTransactionCount: number; + measuredTransactionCount: number; + bucketCounts: Record; +} + +const initialState: GlobalState = createInitialState(); + +export const globalStateSlice = createSlice({ + name: 'globalState', + initialState, + reducers: { + insertState: ( + state, + action: PayloadAction<{ + warehouseCount: number; + measureStartMs: number; + measureEndMs: number; + runStartMs: number; + runEndMs: number; + }> + ) => { + console.log('State inserted, updating global state'); + const payload = action.payload; + state.isReady = true; + state.warehouses = payload.warehouseCount; + state.measureStartMs = payload.measureStartMs; + state.measureEndMs = payload.measureEndMs; + state.runStartMs = payload.runStartMs; + state.runEndMs = payload.runEndMs; + recalculateCounts(state); + }, + deleteState: () => { + console.log('State deleted, resetting to initial state'); + return createInitialState(); + }, + upsertTxnBucket: ( + state, + action: PayloadAction<{ + bucketStartMs: number; + count: number; + }> + ) => { + const payload = action.payload; + state.bucketCounts[payload.bucketStartMs] = payload.count; + recalculateCounts(state); + }, + removeTxnBucket: (state, action: PayloadAction<{ bucketStartMs: number }>) => { + delete state.bucketCounts[action.payload.bucketStartMs]; + recalculateCounts(state); + }, + }, +}); + +export const { insertState, deleteState, upsertTxnBucket, removeTxnBucket } = + globalStateSlice.actions; + +export default globalStateSlice.reducer; diff --git a/tools/tpcc-dashboard/src/hooks.ts b/tools/tpcc-dashboard/src/hooks.ts new file mode 100644 index 00000000000..2cdf69cae23 --- /dev/null +++ b/tools/tpcc-dashboard/src/hooks.ts @@ -0,0 +1,5 @@ +import { useDispatch, useSelector } from 'react-redux'; +import type { RootState, AppDispatch } from './store'; + +export const useAppDispatch = useDispatch.withTypes(); +export const useAppSelector = useSelector.withTypes(); diff --git a/tools/tpcc-dashboard/src/main.tsx b/tools/tpcc-dashboard/src/main.tsx index 4143347ccb2..2cc90fd1824 100644 --- a/tools/tpcc-dashboard/src/main.tsx +++ b/tools/tpcc-dashboard/src/main.tsx @@ -1,12 +1,14 @@ -import { StrictMode } from 'react'; import { createRoot } from 'react-dom/client'; import App from './App.tsx'; import { ConnectedGuard } from './ConnectedGuard.tsx'; +import { Provider } from 'react-redux'; +import { store } from './store.ts'; +import './style.css'; createRoot(document.getElementById('root')!).render( - - + + - - + + ); diff --git a/tools/tpcc-dashboard/src/module_bindings/index.ts b/tools/tpcc-dashboard/src/module_bindings/index.ts index 0d0ce76fa10..5281ad76b8b 100644 --- a/tools/tpcc-dashboard/src/module_bindings/index.ts +++ b/tools/tpcc-dashboard/src/module_bindings/index.ts @@ -1,7 +1,7 @@ // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. -// This was generated using spacetimedb cli version 2.1.0 (commit 6981f48b4bc1a71c8dd9bdfe5a2c343f6370243d). +// This was generated using spacetimedb cli version 2.1.0 (commit 7df4044d896459c2a6ca5b9e2c716052aa39e88a). /* eslint-disable */ /* tslint:disable */ @@ -35,13 +35,16 @@ import { // Import all reducer arg schemas import ClearStateReducer from "./clear_state_reducer"; -import RegisterCompletedOrderReducer from "./register_completed_order_reducer"; +import RecordTxnReducer from "./record_txn_reducer"; +import RecordTxnBucketReducer from "./record_txn_bucket_reducer"; import ResetReducer from "./reset_reducer"; // Import all procedure arg schemas // Import all table schema definitions import StateRow from "./state_table"; +import TxnRow from "./txn_table"; +import TxnBucketRow from "./txn_bucket_table"; /** Type-only namespace exports for generated type groups. */ @@ -58,12 +61,35 @@ const tablesSchema = __schema({ { name: 'state_id_key', constraint: 'unique', columns: ['id'] }, ], }, StateRow), + txn: __table({ + name: 'txn', + indexes: [ + { accessor: 'id', name: 'txn_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + ], + constraints: [ + { name: 'txn_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, TxnRow), + txn_bucket: __table({ + name: 'txn_bucket', + indexes: [ + { accessor: 'bucket_start_ms', name: 'txn_bucket_bucket_start_ms_idx_btree', algorithm: 'btree', columns: [ + 'bucketStartMs', + ] }, + ], + constraints: [ + { name: 'txn_bucket_bucket_start_ms_key', constraint: 'unique', columns: ['bucketStartMs'] }, + ], + }, TxnBucketRow), }); /** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */ const reducersSchema = __reducers( __reducerSchema("clear_state", ClearStateReducer), - __reducerSchema("register_completed_order", RegisterCompletedOrderReducer), + __reducerSchema("record_txn", RecordTxnReducer), + __reducerSchema("record_txn_bucket", RecordTxnBucketReducer), __reducerSchema("reset", ResetReducer), ); diff --git a/tools/tpcc-dashboard/src/module_bindings/register_completed_order_reducer.ts b/tools/tpcc-dashboard/src/module_bindings/record_txn_bucket_reducer.ts similarity index 100% rename from tools/tpcc-dashboard/src/module_bindings/register_completed_order_reducer.ts rename to tools/tpcc-dashboard/src/module_bindings/record_txn_bucket_reducer.ts diff --git a/tools/tpcc-dashboard/src/module_bindings/record_txn_reducer.ts b/tools/tpcc-dashboard/src/module_bindings/record_txn_reducer.ts new file mode 100644 index 00000000000..a6195f0aa10 --- /dev/null +++ b/tools/tpcc-dashboard/src/module_bindings/record_txn_reducer.ts @@ -0,0 +1,15 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + latencyMs: __t.u16(), +}; diff --git a/tools/tpcc-dashboard/src/module_bindings/reset_reducer.ts b/tools/tpcc-dashboard/src/module_bindings/reset_reducer.ts index 73dfa1c32e6..0b0d28699d8 100644 --- a/tools/tpcc-dashboard/src/module_bindings/reset_reducer.ts +++ b/tools/tpcc-dashboard/src/module_bindings/reset_reducer.ts @@ -11,6 +11,7 @@ import { } from "spacetimedb"; export default { + warehouseCount: __t.u64(), warmupDurationMs: __t.u64(), measureStartMs: __t.u64(), measureEndMs: __t.u64(), diff --git a/tools/tpcc-dashboard/src/module_bindings/state_table.ts b/tools/tpcc-dashboard/src/module_bindings/state_table.ts index 954c2bd105a..b4021f845e3 100644 --- a/tools/tpcc-dashboard/src/module_bindings/state_table.ts +++ b/tools/tpcc-dashboard/src/module_bindings/state_table.ts @@ -16,6 +16,5 @@ export default __t.row({ runEndMs: __t.u64().name("run_end_ms"), measureStartMs: __t.u64().name("measure_start_ms"), measureEndMs: __t.u64().name("measure_end_ms"), - orderCount: __t.u64().name("order_count"), - measurementTimeMs: __t.u64().name("measurement_time_ms"), + warehouseCount: __t.u64().name("warehouse_count"), }); diff --git a/tools/tpcc-dashboard/src/module_bindings/txn_bucket_table.ts b/tools/tpcc-dashboard/src/module_bindings/txn_bucket_table.ts new file mode 100644 index 00000000000..702c9a51d58 --- /dev/null +++ b/tools/tpcc-dashboard/src/module_bindings/txn_bucket_table.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + bucketStartMs: __t.u64().primaryKey().name("bucket_start_ms"), + count: __t.u64(), +}); diff --git a/tools/tpcc-dashboard/src/module_bindings/txn_table.ts b/tools/tpcc-dashboard/src/module_bindings/txn_table.ts new file mode 100644 index 00000000000..aac60ff640c --- /dev/null +++ b/tools/tpcc-dashboard/src/module_bindings/txn_table.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + measurementTimeMs: __t.u64().name("measurement_time_ms"), + latencyMs: __t.u16().name("latency_ms"), +}); diff --git a/tools/tpcc-dashboard/src/module_bindings/types.ts b/tools/tpcc-dashboard/src/module_bindings/types.ts index 94c205542c6..37421c0c85a 100644 --- a/tools/tpcc-dashboard/src/module_bindings/types.ts +++ b/tools/tpcc-dashboard/src/module_bindings/types.ts @@ -16,8 +16,20 @@ export const State = __t.object("State", { runEndMs: __t.u64(), measureStartMs: __t.u64(), measureEndMs: __t.u64(), - orderCount: __t.u64(), - measurementTimeMs: __t.u64(), + warehouseCount: __t.u64(), }); export type State = __Infer; +export const Txn = __t.object("Txn", { + id: __t.u64(), + measurementTimeMs: __t.u64(), + latencyMs: __t.u16(), +}); +export type Txn = __Infer; + +export const TxnBucket = __t.object("TxnBucket", { + bucketStartMs: __t.u64(), + count: __t.u64(), +}); +export type TxnBucket = __Infer; + diff --git a/tools/tpcc-dashboard/src/module_bindings/types/reducers.ts b/tools/tpcc-dashboard/src/module_bindings/types/reducers.ts index 81979b5b401..ee9bac76e78 100644 --- a/tools/tpcc-dashboard/src/module_bindings/types/reducers.ts +++ b/tools/tpcc-dashboard/src/module_bindings/types/reducers.ts @@ -7,10 +7,12 @@ import { type Infer as __Infer } from "spacetimedb"; // Import all reducer arg schemas import ClearStateReducer from "../clear_state_reducer"; -import RegisterCompletedOrderReducer from "../register_completed_order_reducer"; +import RecordTxnReducer from "../record_txn_reducer"; +import RecordTxnBucketReducer from "../record_txn_bucket_reducer"; import ResetReducer from "../reset_reducer"; export type ClearStateParams = __Infer; -export type RegisterCompletedOrderParams = __Infer; +export type RecordTxnParams = __Infer; +export type RecordTxnBucketParams = __Infer; export type ResetParams = __Infer; diff --git a/tools/tpcc-dashboard/src/store.ts b/tools/tpcc-dashboard/src/store.ts new file mode 100644 index 00000000000..58734d0ee73 --- /dev/null +++ b/tools/tpcc-dashboard/src/store.ts @@ -0,0 +1,11 @@ +import { configureStore } from '@reduxjs/toolkit'; +import globalStateReducer from './features/globalState'; + +export const store = configureStore({ + reducer: { + globalState: globalStateReducer, + }, +}); + +export type RootState = ReturnType; +export type AppDispatch = typeof store.dispatch; diff --git a/tools/tpcc-dashboard/src/style.css b/tools/tpcc-dashboard/src/style.css new file mode 100644 index 00000000000..efe79c5b7d0 --- /dev/null +++ b/tools/tpcc-dashboard/src/style.css @@ -0,0 +1,71 @@ +:root { + --shade-3: #122129; + --shade-7: #0b1114; + --shade-6: #0e161a; + + --color-green: #4cf490; + --color-white: #d7d8d9; + + --color-neutral-4: #6f7987; + + --background-color: var(--shade-7); + --surface-color: var(--shade-6); + + --font-inter: 'Inter Variable', sans-serif; + --font-source: 'Source Code Pro Variable', monospace; + --font-ibm: 'IBM Plex Mono', monospace; +} + +html { + display: flex; + flex-direction: column; + height: 100vh; + background-color: var(--background-color); +} + +p { + margin: 0; + padding: 0; +} + +.heading-7 { + color: var(--color-white); + + font-family: var(--font-inter); + font-size: 16.8px; + font-style: normal; + font-weight: 500; + line-height: 20.16px; + letter-spacing: -0.672px; +} + +.value-1 { + color: var(--color-green); + + font-family: var(--font-source); + font-size: 24.2px; + font-style: normal; + font-weight: 400; + line-height: 24.2px; +} + +.value-3 { + color: var(--color-neutral-4); + + font-family: var(--font-inter); + font-size: 14px; + font-style: normal; + font-weight: 400; + line-height: 21px; + letter-spacing: -0.14px; +} + +.tagline-2 { + font-family: var(--font-source); + font-size: 11.7px; + font-style: normal; + font-weight: 400; + line-height: 11.7px; + letter-spacing: 0.936px; + text-transform: uppercase; +} diff --git a/tools/tpcc-runner/src/config.rs b/tools/tpcc-runner/src/config.rs index a624663bd8a..43652b7b83c 100644 --- a/tools/tpcc-runner/src/config.rs +++ b/tools/tpcc-runner/src/config.rs @@ -43,6 +43,8 @@ pub struct LoadConfig { pub load_parallelism: usize, pub batch_size: usize, pub reset: bool, + pub warehouse_id_offset: u32, + pub skip_items: bool, } #[derive(Debug, Clone)] @@ -90,6 +92,15 @@ pub struct LoadArgs { pub batch_size: Option, #[arg(long)] pub reset: Option, + /// Offset added to all warehouse IDs for this load. Use when adding warehouses + /// to a database that already has data (e.g. set to 70 to load warehouses 71-140 + /// into a database that already has warehouses 1-70). + #[arg(long)] + pub warehouse_id_offset: Option, + /// Skip loading the global Items table. Use together with --warehouse-id-offset + /// when adding warehouses to an existing database. + #[arg(long)] + pub skip_items: Option, } #[derive(Debug, Clone, Args)] @@ -188,6 +199,8 @@ struct FileLoadConfig { load_parallelism: Option, batch_size: Option, reset: Option, + warehouse_id_offset: Option, + skip_items: Option, } #[derive(Debug, Clone, Default, Deserialize)] @@ -284,6 +297,8 @@ impl LoadArgs { load_parallelism: load_parallelism.min(usize::try_from(num_databases).unwrap_or(usize::MAX)), batch_size, reset: self.reset.or(file.load.reset).unwrap_or(true), + warehouse_id_offset: self.warehouse_id_offset.or(file.load.warehouse_id_offset).unwrap_or(0), + skip_items: self.skip_items.or(file.load.skip_items).unwrap_or(false), }) } } diff --git a/tools/tpcc-runner/src/coordinator.rs b/tools/tpcc-runner/src/coordinator.rs index 5b1e5bb5ce1..6dc11630c2d 100644 --- a/tools/tpcc-runner/src/coordinator.rs +++ b/tools/tpcc-runner/src/coordinator.rs @@ -11,7 +11,7 @@ use std::time::Duration; use crate::config::CoordinatorConfig; use crate::metrics_module_bindings::reset_reducer::reset; -use crate::metrics_module_client::connect_metrics_module; +use crate::metrics_module_client::connect_metrics_module_async; use crate::protocol::{ DriverAssignment, RegisterDriverRequest, RegisterDriverResponse, RunSchedule, ScheduleResponse, SubmitSummaryRequest, @@ -68,37 +68,43 @@ async fn register_driver( State(state): State, Json(request): Json, ) -> Json { - let mut inner = state.inner.lock(); - let (assignment, is_new_registration) = match inner.registrations.get(&request.driver_id) { - Some(existing) => (existing.assignment.clone(), false), - None => { - if inner.registration_order.len() >= inner.config.expected_drivers { - return Json(RegisterDriverResponse { - accepted: false, - assignment: None, - }); + let (assignment, is_new_registration, registered, expected_drivers) = { + let mut inner = state.inner.lock(); + let expected_drivers = inner.config.expected_drivers; + match inner.registrations.get(&request.driver_id) { + Some(existing) => { + let registered = inner.registrations.len(); + (existing.assignment.clone(), false, registered, expected_drivers) + } + None => { + if inner.registration_order.len() >= expected_drivers { + return Json(RegisterDriverResponse { + accepted: false, + assignment: None, + }); + } + let index = inner.registration_order.len(); + let assignment = assignment_for_index(&inner.config, index); + inner.registration_order.push(request.driver_id.clone()); + inner.registrations.insert( + request.driver_id.clone(), + DriverRegistration { + assignment: assignment.clone(), + }, + ); + let registered = inner.registrations.len(); + (assignment, true, registered, expected_drivers) } - let index = inner.registration_order.len(); - let assignment = assignment_for_index(&inner.config, index); - inner.registration_order.push(request.driver_id.clone()); - inner.registrations.insert( - request.driver_id.clone(), - DriverRegistration { - assignment: assignment.clone(), - }, - ); - (assignment, true) } }; - maybe_create_schedule(&mut inner); - let registered = inner.registrations.len(); + maybe_create_schedule(&state).await; let warehouse_end = assignment_end(&assignment); if is_new_registration { log::info!( "driver {} registered and ready ({}/{}): warehouses {}..={} ({} warehouse(s))", request.driver_id, registered, - inner.config.expected_drivers, + expected_drivers, assignment.warehouse_start, warehouse_end, assignment.driver_warehouse_count @@ -168,35 +174,58 @@ async fn submit_summary( Ok(Json(aggregate)) } -fn maybe_create_schedule(inner: &mut CoordinatorState) { - if inner.schedule.is_some() || inner.registrations.len() < inner.config.expected_drivers { - return; - } +async fn maybe_create_schedule(state: &AppState) { + // Check whether schedule creation is needed, and grab config, without holding the lock + // during the async metrics module connection below. + let config = { + let inner = state.inner.lock(); + if inner.schedule.is_some() || inner.registrations.len() < inner.config.expected_drivers { + return; + } + inner.config.clone() + }; + let warmup_start_ms = now_millis() + 2_000; - let measure_start_ms = warmup_start_ms + (inner.config.warmup_secs * 1_000); - let measure_end_ms = measure_start_ms + (inner.config.measure_secs * 1_000); + let measure_start_ms = warmup_start_ms + (config.warmup_secs * 1_000); + let measure_end_ms = measure_start_ms + (config.measure_secs * 1_000); - let metrics_client = connect_metrics_module(&inner.config.connection).unwrap(); - let _ = metrics_client - .reducers - .reset(inner.config.warmup_secs * 1000, measure_start_ms, measure_end_ms); + let metrics_client = match connect_metrics_module_async(&config.connection).await { + Ok(client) => client, + Err(e) => { + log::error!("failed to connect to metrics module: {e:#}"); + return; + } + }; + let _ = metrics_client.reducers.reset( + config.warehouses as u64, + config.warmup_secs * 1000, + measure_start_ms, + measure_end_ms, + ); let schedule = RunSchedule { - run_id: inner.config.run_id.clone(), + run_id: config.run_id.clone(), warmup_start_ms, measure_start_ms, measure_end_ms, stop_ms: measure_end_ms, }; + + let mut inner = state.inner.lock(); + if inner.schedule.is_some() { + // Another concurrent registration call already created the schedule. + return; + } inner.schedule = Some(schedule.clone()); log::info!( "all {} driver(s) registered; schedule ready for run {} (warmup_start_ms={} measure_start_ms={} measure_end_ms={})", - inner.config.expected_drivers, - inner.config.run_id, + config.expected_drivers, + config.run_id, warmup_start_ms, measure_start_ms, measure_end_ms ); + drop(inner); tokio::spawn(log_schedule_events(schedule)); } diff --git a/tools/tpcc-runner/src/driver.rs b/tools/tpcc-runner/src/driver.rs index cb937f9d8b3..0b7926ecfae 100644 --- a/tools/tpcc-runner/src/driver.rs +++ b/tools/tpcc-runner/src/driver.rs @@ -2,7 +2,6 @@ use anyhow::{anyhow, bail, Context, Result}; use rand::{rngs::StdRng, Rng, SeedableRng}; use std::collections::BTreeMap; use std::fs; -use std::future::Future; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; @@ -11,8 +10,7 @@ use tokio::task::JoinSet; use crate::client::{expect_ok, ModuleClient}; use crate::config::{default_run_id, DriverConfig}; -use crate::metrics_module_bindings::register_completed_order; -use crate::metrics_module_bindings::DbConnection as MetricsDbConnection; +use crate::metrics_module_bindings::{record_txn_bucket, DbConnection as MetricsDbConnection}; use crate::metrics_module_client::connect_metrics_module_async; use crate::module_bindings::*; use crate::protocol::{ @@ -24,9 +22,6 @@ use crate::summary::{ use crate::topology::DatabaseTopology; use crate::tpcc::*; -const REDUCER_CALL_MAX_ATTEMPTS: u32 = 3; -const REDUCER_RETRY_DELAY_MS: u64 = 250; - struct TerminalRuntime { config: DriverConfig, client: Arc, @@ -52,16 +47,6 @@ struct TransactionContext<'a> { request_ids: &'a AtomicU64, } -enum ReducerCallOutcome { - Succeeded(T), - Failed(anyhow::Error), -} - -enum NewOrderExecution { - Committed, - RolledBack(String), -} - pub async fn run(config: DriverConfig) -> Result<()> { let (config, schedule) = resolve_driver_setup(config).await?; let run_id = schedule.run_id.clone(); @@ -265,7 +250,7 @@ async fn run_terminal(runtime: TerminalRuntime) -> Result<()> { // Some metrics depend on knowing all completed orders, even outside the // measurement window if record.kind == TransactionKind::NewOrder && record.success { - let _ = metrics_client.reducers.register_completed_order(); + let _ = metrics_client.reducers.record_txn_bucket(); } if record.timestamp_ms >= schedule.measure_start_ms && record.timestamp_ms < schedule.measure_end_ms { @@ -273,8 +258,9 @@ async fn run_terminal(runtime: TerminalRuntime) -> Result<()> { } } Err(err) => { - abort.store(true, Ordering::Relaxed); - return Err(err); + log::error!( + "terminal task error: {err:#}", + ); } } @@ -293,40 +279,76 @@ async fn execute_transaction( started_ms: u64, ) -> Result { match kind { - TransactionKind::NewOrder => execute_new_order(context, rng, started_ms).await, - TransactionKind::Payment => execute_payment(context, rng, started_ms).await, - TransactionKind::OrderStatus => execute_order_status(context, rng, started_ms).await, - TransactionKind::Delivery => execute_delivery(context, rng, started_ms).await, - TransactionKind::StockLevel => execute_stock_level(context, rng, started_ms).await, + TransactionKind::NewOrder => { + execute_new_order( + context.client, + context.config.warehouse_count, + context.assignment, + context.constants, + rng, + started_ms, + ) + .await + } + TransactionKind::Payment => { + execute_payment( + context.client, + context.config.warehouse_count, + context.assignment, + context.constants, + rng, + started_ms, + ) + .await + } + TransactionKind::OrderStatus => { + execute_order_status(context.client, context.assignment, context.constants, rng, started_ms).await + } + TransactionKind::Delivery => { + execute_delivery( + context.client, + context.run_id, + context.driver_id, + context.assignment, + context.request_ids, + rng, + started_ms, + ) + .await + } + TransactionKind::StockLevel => execute_stock_level(context.client, context.assignment, rng, started_ms).await, } } async fn execute_new_order( - context: &TransactionContext<'_>, + client: &ModuleClient, + warehouse_count: u32, + assignment: &TerminalAssignment, + constants: &RunConstants, rng: &mut StdRng, started_ms: u64, ) -> Result { - let customer_id = customer_id(rng, context.constants); + let customer_id = customer_id(rng, constants); let line_count = rng.random_range(5..=15); let invalid_line = rng.random_bool(0.01); let mut order_lines = Vec::with_capacity(line_count); let mut remote_order_line_count = 0u32; for idx in 0..line_count { - let remote = context.config.warehouse_count > 1 && rng.random_bool(0.01); + let remote = warehouse_count > 1 && rng.random_bool(0.01); let supply_w_id = if remote { remote_order_line_count += 1; - let mut remote = context.assignment.warehouse_id; - while remote == context.assignment.warehouse_id { - remote = rng.random_range(1..=context.config.warehouse_count); + let mut remote = assignment.warehouse_id; + while remote == assignment.warehouse_id { + remote = rng.random_range(1..=warehouse_count); } remote } else { - context.assignment.warehouse_id + assignment.warehouse_id }; let item_id = if invalid_line && idx + 1 == line_count { ITEMS + 1 } else { - item_id(rng, context.constants) + item_id(rng, constants) }; order_lines.push(NewOrderLineInput { item_id, @@ -335,31 +357,19 @@ async fn execute_new_order( }); } - let result = retry_reducer_call(context, "new_order", || { - let order_lines = order_lines.clone(); - async move { - match context - .client - .new_order_async( - context.assignment.warehouse_id, - context.assignment.district_id, - customer_id, - order_lines, - ) - .await? - { - Ok(_) => Ok(NewOrderExecution::Committed), - Err(message) if invalid_line => Ok(NewOrderExecution::RolledBack(message)), - Err(message) => Err(anyhow!("new_order failed: {}", message)), - } - } - }) - .await; + let result = client + .new_order_async( + assignment.warehouse_id, + assignment.district_id, + customer_id, + order_lines, + ) + .await?; let finished_ms = crate::summary::now_millis(); match result { - ReducerCallOutcome::Succeeded(NewOrderExecution::Committed) => Ok(TransactionRecord { + Ok(_) => Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: context.assignment.terminal_id, + terminal_id: assignment.terminal_id, kind: TransactionKind::NewOrder, success: true, latency_ms: finished_ms.saturating_sub(started_ms), @@ -370,9 +380,9 @@ async fn execute_new_order( remote_order_line_count, detail: None, }), - ReducerCallOutcome::Succeeded(NewOrderExecution::RolledBack(message)) => Ok(TransactionRecord { + Err(message) if invalid_line => Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: context.assignment.terminal_id, + terminal_id: assignment.terminal_id, kind: TransactionKind::NewOrder, success: false, latency_ms: finished_ms.saturating_sub(started_ms), @@ -383,282 +393,177 @@ async fn execute_new_order( remote_order_line_count, detail: Some(message), }), - ReducerCallOutcome::Failed(err) => Ok(TransactionRecord { - timestamp_ms: finished_ms, - terminal_id: context.assignment.terminal_id, - kind: TransactionKind::NewOrder, - success: false, - latency_ms: finished_ms.saturating_sub(started_ms), - rollback: false, - remote: false, - by_last_name: false, - order_line_count: line_count as u32, - remote_order_line_count, - detail: Some(format!("{err:#}")), - }), + Err(message) => bail!( + "unexpected new_order failure for terminal {}: {}", + assignment.terminal_id, + message + ), } } async fn execute_payment( - context: &TransactionContext<'_>, + client: &ModuleClient, + warehouse_count: u32, + assignment: &TerminalAssignment, + constants: &RunConstants, rng: &mut StdRng, started_ms: u64, ) -> Result { - let remote = context.config.warehouse_count > 1 && rng.random_bool(0.15); + let remote = warehouse_count > 1 && rng.random_bool(0.15); let c_w_id = if remote { - let mut other = context.assignment.warehouse_id; - while other == context.assignment.warehouse_id { - other = rng.random_range(1..=context.config.warehouse_count); + let mut other = assignment.warehouse_id; + while other == assignment.warehouse_id { + other = rng.random_range(1..=warehouse_count); } other } else { - context.assignment.warehouse_id + assignment.warehouse_id }; let c_d_id = if remote { rng.random_range(1..=DISTRICTS_PER_WAREHOUSE) } else { - context.assignment.district_id + assignment.district_id }; let by_last_name = rng.random_bool(0.60); let selector = if by_last_name { - CustomerSelector::ByLastName(customer_last_name(rng, context.constants)) + CustomerSelector::ByLastName(customer_last_name(rng, constants)) } else { - CustomerSelector::ById(customer_id(rng, context.constants)) + CustomerSelector::ById(customer_id(rng, constants)) }; let amount_cents = rng.random_range(100..=500_000); - let result = retry_reducer_call(context, "payment", || { - let selector = selector.clone(); - async move { - let _ = expect_ok( - "payment", - context - .client - .payment_async( - context.assignment.warehouse_id, - context.assignment.district_id, - c_w_id, - c_d_id, - selector, - amount_cents, - ) - .await, - )?; - Ok(()) - } - }) - .await; + let finished = expect_ok( + "payment", + client + .payment_async( + assignment.warehouse_id, + assignment.district_id, + c_w_id, + c_d_id, + selector, + amount_cents, + ) + .await, + )?; + let _ = finished; let finished_ms = crate::summary::now_millis(); - let (success, detail) = match result { - ReducerCallOutcome::Succeeded(()) => (true, None), - ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), - }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: context.assignment.terminal_id, + terminal_id: assignment.terminal_id, kind: TransactionKind::Payment, - success, + success: true, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote, by_last_name, order_line_count: 0, remote_order_line_count: 0, - detail, + detail: None, }) } async fn execute_order_status( - context: &TransactionContext<'_>, + client: &ModuleClient, + assignment: &TerminalAssignment, + constants: &RunConstants, rng: &mut StdRng, started_ms: u64, ) -> Result { let by_last_name = rng.random_bool(0.60); let selector = if by_last_name { - CustomerSelector::ByLastName(customer_last_name(rng, context.constants)) + CustomerSelector::ByLastName(customer_last_name(rng, constants)) } else { - CustomerSelector::ById(customer_id(rng, context.constants)) + CustomerSelector::ById(customer_id(rng, constants)) }; - let result = retry_reducer_call(context, "order_status", || { - let selector = selector.clone(); - async move { - let _ = expect_ok( - "order_status", - context - .client - .order_status_async( - context.assignment.warehouse_id, - context.assignment.district_id, - selector, - ) - .await, - )?; - Ok(()) - } - }) - .await; + let _ = expect_ok( + "order_status", + client + .order_status_async(assignment.warehouse_id, assignment.district_id, selector) + .await, + )?; let finished_ms = crate::summary::now_millis(); - let (success, detail) = match result { - ReducerCallOutcome::Succeeded(()) => (true, None), - ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), - }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: context.assignment.terminal_id, + terminal_id: assignment.terminal_id, kind: TransactionKind::OrderStatus, - success, + success: true, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote: false, by_last_name, order_line_count: 0, remote_order_line_count: 0, - detail, + detail: None, }) } async fn execute_delivery( - context: &TransactionContext<'_>, + client: &ModuleClient, + run_id: &str, + driver_id: &str, + assignment: &TerminalAssignment, + request_ids: &AtomicU64, rng: &mut StdRng, started_ms: u64, ) -> Result { - let request_id = context.request_ids.fetch_add(1, Ordering::Relaxed); - let carrier_id = rng.random_range(1..=10); - let result = retry_reducer_call(context, "queue_delivery", || async move { - let _ = expect_ok( - "queue_delivery", - context - .client - .queue_delivery_async( - context.run_id.to_string(), - context.driver_id.to_string(), - context.assignment.terminal_id, - request_id, - context.assignment.warehouse_id, - carrier_id, - ) - .await, - )?; - Ok(()) - }) - .await; + let request_id = request_ids.fetch_add(1, Ordering::Relaxed); + let _ = expect_ok( + "queue_delivery", + client + .queue_delivery_async( + run_id.to_string(), + driver_id.to_string(), + assignment.terminal_id, + request_id, + assignment.warehouse_id, + rng.random_range(1..=10), + ) + .await, + )?; let finished_ms = crate::summary::now_millis(); - let (success, detail) = match result { - ReducerCallOutcome::Succeeded(()) => (true, None), - ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), - }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: context.assignment.terminal_id, + terminal_id: assignment.terminal_id, kind: TransactionKind::Delivery, - success, + success: true, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote: false, by_last_name: false, order_line_count: 0, remote_order_line_count: 0, - detail, + detail: None, }) } async fn execute_stock_level( - context: &TransactionContext<'_>, + client: &ModuleClient, + assignment: &TerminalAssignment, rng: &mut StdRng, started_ms: u64, ) -> Result { let threshold = rng.random_range(10..=20); - let result = retry_reducer_call(context, "stock_level", || async move { - let _ = expect_ok( - "stock_level", - context - .client - .stock_level_async( - context.assignment.warehouse_id, - context.assignment.district_id, - threshold, - ) - .await, - )?; - Ok(()) - }) - .await; + let _ = expect_ok( + "stock_level", + client + .stock_level_async(assignment.warehouse_id, assignment.district_id, threshold) + .await, + )?; let finished_ms = crate::summary::now_millis(); - let (success, detail) = match result { - ReducerCallOutcome::Succeeded(()) => (true, None), - ReducerCallOutcome::Failed(err) => (false, Some(format!("{err:#}"))), - }; Ok(TransactionRecord { timestamp_ms: finished_ms, - terminal_id: context.assignment.terminal_id, + terminal_id: assignment.terminal_id, kind: TransactionKind::StockLevel, - success, + success: true, latency_ms: finished_ms.saturating_sub(started_ms), rollback: false, remote: false, by_last_name: false, order_line_count: 0, remote_order_line_count: 0, - detail, + detail: None, }) } -async fn retry_reducer_call( - context: &TransactionContext<'_>, - reducer_name: &'static str, - mut call: F, -) -> ReducerCallOutcome -where - F: FnMut() -> Fut, - Fut: Future>, -{ - let mut last_error = None; - - for attempt in 1..=REDUCER_CALL_MAX_ATTEMPTS { - match call().await { - Ok(value) => { - if attempt > 1 { - log::info!( - "driver {} terminal {} reducer {} succeeded on attempt {}/{}", - context.driver_id, - context.assignment.terminal_id, - reducer_name, - attempt, - REDUCER_CALL_MAX_ATTEMPTS - ); - } - return ReducerCallOutcome::Succeeded(value); - } - Err(err) => { - if attempt == REDUCER_CALL_MAX_ATTEMPTS { - log::error!( - "driver {} terminal {} reducer {} failed after {} attempt(s): {err:#}", - context.driver_id, - context.assignment.terminal_id, - reducer_name, - REDUCER_CALL_MAX_ATTEMPTS - ); - last_error = Some(err); - break; - } - - log::warn!( - "driver {} terminal {} reducer {} failed on attempt {}/{}: {err:#}; retrying in {}ms", - context.driver_id, - context.assignment.terminal_id, - reducer_name, - attempt, - REDUCER_CALL_MAX_ATTEMPTS, - REDUCER_RETRY_DELAY_MS - ); - tokio::time::sleep(Duration::from_millis(REDUCER_RETRY_DELAY_MS)).await; - last_error = Some(err); - } - } - } - - ReducerCallOutcome::Failed(last_error.unwrap_or_else(|| anyhow!("{} failed without an error", reducer_name))) -} - async fn resolve_driver_setup(config: DriverConfig) -> Result<(DriverConfig, RunSchedule)> { if let Some(coordinator_url) = &config.coordinator_url { const REGISTER_ATTEMPTS: u32 = 5; diff --git a/tools/tpcc-runner/src/loader.rs b/tools/tpcc-runner/src/loader.rs index 559aa819d79..3471ad89f9c 100644 --- a/tools/tpcc-runner/src/loader.rs +++ b/tools/tpcc-runner/src/loader.rs @@ -132,6 +132,8 @@ fn build_load_request( database_number, num_databases: config.num_databases, warehouses_per_database: config.warehouses_per_database, + warehouse_id_offset: config.warehouse_id_offset, + skip_items: config.skip_items, batch_size: u32::try_from(config.batch_size).context("batch_size exceeds u32")?, seed: LOAD_SEED, load_c_last, diff --git a/tools/tpcc-runner/src/metrics_module_bindings/clear_state_reducer.rs b/tools/tpcc-runner/src/metrics_module_bindings/clear_state_reducer.rs index fffd58d13d3..df42291348e 100644 --- a/tools/tpcc-runner/src/metrics_module_bindings/clear_state_reducer.rs +++ b/tools/tpcc-runner/src/metrics_module_bindings/clear_state_reducer.rs @@ -56,6 +56,7 @@ impl clear_state for super::RemoteReducers { + Send + 'static, ) -> __sdk::Result<()> { - self.imp.invoke_reducer_with_callback(ClearStateArgs {}, callback) + self.imp + .invoke_reducer_with_callback::<_, ()>(ClearStateArgs {}, callback) } } diff --git a/tools/tpcc-runner/src/metrics_module_bindings/mod.rs b/tools/tpcc-runner/src/metrics_module_bindings/mod.rs index 6322519697f..0e35a1d7729 100644 --- a/tools/tpcc-runner/src/metrics_module_bindings/mod.rs +++ b/tools/tpcc-runner/src/metrics_module_bindings/mod.rs @@ -1,22 +1,32 @@ // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. -// This was generated using spacetimedb cli version 2.1.0 (commit 6981f48b4bc1a71c8dd9bdfe5a2c343f6370243d). +// This was generated using spacetimedb cli version 2.1.0 (commit 7df4044d896459c2a6ca5b9e2c716052aa39e88a). #![allow(unused, clippy::all)] use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; pub mod clear_state_reducer; -pub mod register_completed_order_reducer; +pub mod record_txn_bucket_reducer; +pub mod record_txn_reducer; pub mod reset_reducer; pub mod state_table; pub mod state_type; +pub mod txn_bucket_table; +pub mod txn_bucket_type; +pub mod txn_table; +pub mod txn_type; pub use clear_state_reducer::clear_state; -pub use register_completed_order_reducer::register_completed_order; +pub use record_txn_bucket_reducer::record_txn_bucket; +pub use record_txn_reducer::record_txn; pub use reset_reducer::reset; pub use state_table::*; pub use state_type::State; +pub use txn_bucket_table::*; +pub use txn_bucket_type::TxnBucket; +pub use txn_table::*; +pub use txn_type::Txn; #[derive(Clone, PartialEq, Debug)] @@ -27,8 +37,12 @@ pub use state_type::State; pub enum Reducer { ClearState, - RegisterCompletedOrder, + RecordTxn { + latency_ms: u16, + }, + RecordTxnBucket, Reset { + warehouse_count: u64, warmup_duration_ms: u64, measure_start_ms: u64, measure_end_ms: u64, @@ -43,7 +57,8 @@ impl __sdk::Reducer for Reducer { fn reducer_name(&self) -> &'static str { match self { Reducer::ClearState => "clear_state", - Reducer::RegisterCompletedOrder => "register_completed_order", + Reducer::RecordTxn { .. } => "record_txn", + Reducer::RecordTxnBucket => "record_txn_bucket", Reducer::Reset { .. } => "reset", _ => unreachable!(), } @@ -52,14 +67,17 @@ impl __sdk::Reducer for Reducer { fn args_bsatn(&self) -> Result, __sats::bsatn::EncodeError> { match self { Reducer::ClearState => __sats::bsatn::to_vec(&clear_state_reducer::ClearStateArgs {}), - Reducer::RegisterCompletedOrder => { - __sats::bsatn::to_vec(®ister_completed_order_reducer::RegisterCompletedOrderArgs {}) - } + Reducer::RecordTxn { latency_ms } => __sats::bsatn::to_vec(&record_txn_reducer::RecordTxnArgs { + latency_ms: latency_ms.clone(), + }), + Reducer::RecordTxnBucket => __sats::bsatn::to_vec(&record_txn_bucket_reducer::RecordTxnBucketArgs {}), Reducer::Reset { + warehouse_count, warmup_duration_ms, measure_start_ms, measure_end_ms, } => __sats::bsatn::to_vec(&reset_reducer::ResetArgs { + warehouse_count: warehouse_count.clone(), warmup_duration_ms: warmup_duration_ms.clone(), measure_start_ms: measure_start_ms.clone(), measure_end_ms: measure_end_ms.clone(), @@ -74,6 +92,8 @@ impl __sdk::Reducer for Reducer { #[doc(hidden)] pub struct DbUpdate { state: __sdk::TableUpdate, + txn: __sdk::TableUpdate, + txn_bucket: __sdk::TableUpdate, } impl TryFrom<__ws::v2::TransactionUpdate> for DbUpdate { @@ -83,6 +103,10 @@ impl TryFrom<__ws::v2::TransactionUpdate> for DbUpdate { for table_update in __sdk::transaction_update_iter_table_updates(raw) { match &table_update.table_name[..] { "state" => db_update.state.append(state_table::parse_table_update(table_update)?), + "txn" => db_update.txn.append(txn_table::parse_table_update(table_update)?), + "txn_bucket" => db_update + .txn_bucket + .append(txn_bucket_table::parse_table_update(table_update)?), unknown => { return Err(__sdk::InternalError::unknown_name("table", unknown, "DatabaseUpdate").into()); @@ -104,6 +128,12 @@ impl __sdk::DbUpdate for DbUpdate { diff.state = cache .apply_diff_to_table::("state", &self.state) .with_updates_by_pk(|row| &row.id); + diff.txn = cache + .apply_diff_to_table::("txn", &self.txn) + .with_updates_by_pk(|row| &row.id); + diff.txn_bucket = cache + .apply_diff_to_table::("txn_bucket", &self.txn_bucket) + .with_updates_by_pk(|row| &row.bucket_start_ms); diff } @@ -114,6 +144,10 @@ impl __sdk::DbUpdate for DbUpdate { "state" => db_update .state .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), + "txn" => db_update.txn.append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), + "txn_bucket" => db_update + .txn_bucket + .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), unknown => { return Err(__sdk::InternalError::unknown_name("table", unknown, "QueryRows").into()); } @@ -128,6 +162,10 @@ impl __sdk::DbUpdate for DbUpdate { "state" => db_update .state .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), + "txn" => db_update.txn.append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), + "txn_bucket" => db_update + .txn_bucket + .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), unknown => { return Err(__sdk::InternalError::unknown_name("table", unknown, "QueryRows").into()); } @@ -142,6 +180,8 @@ impl __sdk::DbUpdate for DbUpdate { #[doc(hidden)] pub struct AppliedDiff<'r> { state: __sdk::TableAppliedDiff<'r, State>, + txn: __sdk::TableAppliedDiff<'r, Txn>, + txn_bucket: __sdk::TableAppliedDiff<'r, TxnBucket>, __unused: std::marker::PhantomData<&'r ()>, } @@ -152,6 +192,8 @@ impl __sdk::InModule for AppliedDiff<'_> { impl<'r> __sdk::AppliedDiff<'r> for AppliedDiff<'r> { fn invoke_row_callbacks(&self, event: &EventContext, callbacks: &mut __sdk::DbCallbacks) { callbacks.invoke_table_row_callbacks::("state", &self.state, event); + callbacks.invoke_table_row_callbacks::("txn", &self.txn, event); + callbacks.invoke_table_row_callbacks::("txn_bucket", &self.txn_bucket, event); } } @@ -810,6 +852,8 @@ impl __sdk::SpacetimeModule for RemoteModule { fn register_tables(client_cache: &mut __sdk::ClientCache) { state_table::register_table(client_cache); + txn_table::register_table(client_cache); + txn_bucket_table::register_table(client_cache); } - const ALL_TABLE_NAMES: &'static [&'static str] = &["state"]; + const ALL_TABLE_NAMES: &'static [&'static str] = &["state", "txn", "txn_bucket"]; } diff --git a/tools/tpcc-runner/src/metrics_module_bindings/register_completed_order_reducer.rs b/tools/tpcc-runner/src/metrics_module_bindings/record_txn_bucket_reducer.rs similarity index 58% rename from tools/tpcc-runner/src/metrics_module_bindings/register_completed_order_reducer.rs rename to tools/tpcc-runner/src/metrics_module_bindings/record_txn_bucket_reducer.rs index 3cf5f048e5a..20c629a9720 100644 --- a/tools/tpcc-runner/src/metrics_module_bindings/register_completed_order_reducer.rs +++ b/tools/tpcc-runner/src/metrics_module_bindings/record_txn_bucket_reducer.rs @@ -6,40 +6,40 @@ use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; #[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] #[sats(crate = __lib)] -pub(super) struct RegisterCompletedOrderArgs {} +pub(super) struct RecordTxnBucketArgs {} -impl From for super::Reducer { - fn from(args: RegisterCompletedOrderArgs) -> Self { - Self::RegisterCompletedOrder +impl From for super::Reducer { + fn from(args: RecordTxnBucketArgs) -> Self { + Self::RecordTxnBucket } } -impl __sdk::InModule for RegisterCompletedOrderArgs { +impl __sdk::InModule for RecordTxnBucketArgs { type Module = super::RemoteModule; } #[allow(non_camel_case_types)] -/// Extension trait for access to the reducer `register_completed_order`. +/// Extension trait for access to the reducer `record_txn_bucket`. /// /// Implemented for [`super::RemoteReducers`]. -pub trait register_completed_order { - /// Request that the remote module invoke the reducer `register_completed_order` to run as soon as possible. +pub trait record_txn_bucket { + /// Request that the remote module invoke the reducer `record_txn_bucket` to run as soon as possible. /// /// This method returns immediately, and errors only if we are unable to send the request. /// The reducer will run asynchronously in the future, /// and this method provides no way to listen for its completion status. - /// /// Use [`register_completed_order:register_completed_order_then`] to run a callback after the reducer completes. - fn register_completed_order(&self) -> __sdk::Result<()> { - self.register_completed_order_then(|_, _| {}) + /// /// Use [`record_txn_bucket:record_txn_bucket_then`] to run a callback after the reducer completes. + fn record_txn_bucket(&self) -> __sdk::Result<()> { + self.record_txn_bucket_then(|_, _| {}) } - /// Request that the remote module invoke the reducer `register_completed_order` to run as soon as possible, + /// Request that the remote module invoke the reducer `record_txn_bucket` to run as soon as possible, /// registering `callback` to run when we are notified that the reducer completed. /// /// This method returns immediately, and errors only if we are unable to send the request. /// The reducer will run asynchronously in the future, /// and its status can be observed with the `callback`. - fn register_completed_order_then( + fn record_txn_bucket_then( &self, callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) @@ -48,8 +48,8 @@ pub trait register_completed_order { ) -> __sdk::Result<()>; } -impl register_completed_order for super::RemoteReducers { - fn register_completed_order_then( +impl record_txn_bucket for super::RemoteReducers { + fn record_txn_bucket_then( &self, callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) @@ -57,6 +57,6 @@ impl register_completed_order for super::RemoteReducers { + 'static, ) -> __sdk::Result<()> { self.imp - .invoke_reducer_with_callback(RegisterCompletedOrderArgs {}, callback) + .invoke_reducer_with_callback::<_, ()>(RecordTxnBucketArgs {}, callback) } } diff --git a/tools/tpcc-runner/src/metrics_module_bindings/record_txn_reducer.rs b/tools/tpcc-runner/src/metrics_module_bindings/record_txn_reducer.rs new file mode 100644 index 00000000000..3ccccd97476 --- /dev/null +++ b/tools/tpcc-runner/src/metrics_module_bindings/record_txn_reducer.rs @@ -0,0 +1,68 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub(super) struct RecordTxnArgs { + pub latency_ms: u16, +} + +impl From for super::Reducer { + fn from(args: RecordTxnArgs) -> Self { + Self::RecordTxn { + latency_ms: args.latency_ms, + } + } +} + +impl __sdk::InModule for RecordTxnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the reducer `record_txn`. +/// +/// Implemented for [`super::RemoteReducers`]. +pub trait record_txn { + /// Request that the remote module invoke the reducer `record_txn` to run as soon as possible. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and this method provides no way to listen for its completion status. + /// /// Use [`record_txn:record_txn_then`] to run a callback after the reducer completes. + fn record_txn(&self, latency_ms: u16) -> __sdk::Result<()> { + self.record_txn_then(latency_ms, |_, _| {}) + } + + /// Request that the remote module invoke the reducer `record_txn` to run as soon as possible, + /// registering `callback` to run when we are notified that the reducer completed. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and its status can be observed with the `callback`. + fn record_txn_then( + &self, + latency_ms: u16, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()>; +} + +impl record_txn for super::RemoteReducers { + fn record_txn_then( + &self, + latency_ms: u16, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()> { + self.imp + .invoke_reducer_with_callback::<_, ()>(RecordTxnArgs { latency_ms }, callback) + } +} diff --git a/tools/tpcc-runner/src/metrics_module_bindings/reset_reducer.rs b/tools/tpcc-runner/src/metrics_module_bindings/reset_reducer.rs index 7c63e35f429..f74c3cd3662 100644 --- a/tools/tpcc-runner/src/metrics_module_bindings/reset_reducer.rs +++ b/tools/tpcc-runner/src/metrics_module_bindings/reset_reducer.rs @@ -7,6 +7,7 @@ use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; #[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] #[sats(crate = __lib)] pub(super) struct ResetArgs { + pub warehouse_count: u64, pub warmup_duration_ms: u64, pub measure_start_ms: u64, pub measure_end_ms: u64, @@ -15,6 +16,7 @@ pub(super) struct ResetArgs { impl From for super::Reducer { fn from(args: ResetArgs) -> Self { Self::Reset { + warehouse_count: args.warehouse_count, warmup_duration_ms: args.warmup_duration_ms, measure_start_ms: args.measure_start_ms, measure_end_ms: args.measure_end_ms, @@ -37,8 +39,20 @@ pub trait reset { /// The reducer will run asynchronously in the future, /// and this method provides no way to listen for its completion status. /// /// Use [`reset:reset_then`] to run a callback after the reducer completes. - fn reset(&self, warmup_duration_ms: u64, measure_start_ms: u64, measure_end_ms: u64) -> __sdk::Result<()> { - self.reset_then(warmup_duration_ms, measure_start_ms, measure_end_ms, |_, _| {}) + fn reset( + &self, + warehouse_count: u64, + warmup_duration_ms: u64, + measure_start_ms: u64, + measure_end_ms: u64, + ) -> __sdk::Result<()> { + self.reset_then( + warehouse_count, + warmup_duration_ms, + measure_start_ms, + measure_end_ms, + |_, _| {}, + ) } /// Request that the remote module invoke the reducer `reset` to run as soon as possible, @@ -49,6 +63,7 @@ pub trait reset { /// and its status can be observed with the `callback`. fn reset_then( &self, + warehouse_count: u64, warmup_duration_ms: u64, measure_start_ms: u64, measure_end_ms: u64, @@ -62,6 +77,7 @@ pub trait reset { impl reset for super::RemoteReducers { fn reset_then( &self, + warehouse_count: u64, warmup_duration_ms: u64, measure_start_ms: u64, measure_end_ms: u64, @@ -70,8 +86,9 @@ impl reset for super::RemoteReducers { + Send + 'static, ) -> __sdk::Result<()> { - self.imp.invoke_reducer_with_callback( + self.imp.invoke_reducer_with_callback::<_, ()>( ResetArgs { + warehouse_count, warmup_duration_ms, measure_start_ms, measure_end_ms, diff --git a/tools/tpcc-runner/src/metrics_module_bindings/state_type.rs b/tools/tpcc-runner/src/metrics_module_bindings/state_type.rs index 98d8a762262..307bdcaaef9 100644 --- a/tools/tpcc-runner/src/metrics_module_bindings/state_type.rs +++ b/tools/tpcc-runner/src/metrics_module_bindings/state_type.rs @@ -12,8 +12,7 @@ pub struct State { pub run_end_ms: u64, pub measure_start_ms: u64, pub measure_end_ms: u64, - pub order_count: u64, - pub measurement_time_ms: u64, + pub warehouse_count: u64, } impl __sdk::InModule for State { @@ -29,8 +28,7 @@ pub struct StateCols { pub run_end_ms: __sdk::__query_builder::Col, pub measure_start_ms: __sdk::__query_builder::Col, pub measure_end_ms: __sdk::__query_builder::Col, - pub order_count: __sdk::__query_builder::Col, - pub measurement_time_ms: __sdk::__query_builder::Col, + pub warehouse_count: __sdk::__query_builder::Col, } impl __sdk::__query_builder::HasCols for State { @@ -42,8 +40,7 @@ impl __sdk::__query_builder::HasCols for State { run_end_ms: __sdk::__query_builder::Col::new(table_name, "run_end_ms"), measure_start_ms: __sdk::__query_builder::Col::new(table_name, "measure_start_ms"), measure_end_ms: __sdk::__query_builder::Col::new(table_name, "measure_end_ms"), - order_count: __sdk::__query_builder::Col::new(table_name, "order_count"), - measurement_time_ms: __sdk::__query_builder::Col::new(table_name, "measurement_time_ms"), + warehouse_count: __sdk::__query_builder::Col::new(table_name, "warehouse_count"), } } } diff --git a/tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_table.rs b/tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_table.rs new file mode 100644 index 00000000000..14b3d56c3cf --- /dev/null +++ b/tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_table.rs @@ -0,0 +1,157 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use super::txn_bucket_type::TxnBucket; +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +/// Table handle for the table `txn_bucket`. +/// +/// Obtain a handle from the [`TxnBucketTableAccess::txn_bucket`] method on [`super::RemoteTables`], +/// like `ctx.db.txn_bucket()`. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.txn_bucket().on_insert(...)`. +pub struct TxnBucketTableHandle<'ctx> { + imp: __sdk::TableHandle, + ctx: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the table `txn_bucket`. +/// +/// Implemented for [`super::RemoteTables`]. +pub trait TxnBucketTableAccess { + #[allow(non_snake_case)] + /// Obtain a [`TxnBucketTableHandle`], which mediates access to the table `txn_bucket`. + fn txn_bucket(&self) -> TxnBucketTableHandle<'_>; +} + +impl TxnBucketTableAccess for super::RemoteTables { + fn txn_bucket(&self) -> TxnBucketTableHandle<'_> { + TxnBucketTableHandle { + imp: self.imp.get_table::("txn_bucket"), + ctx: std::marker::PhantomData, + } + } +} + +pub struct TxnBucketInsertCallbackId(__sdk::CallbackId); +pub struct TxnBucketDeleteCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::Table for TxnBucketTableHandle<'ctx> { + type Row = TxnBucket; + type EventContext = super::EventContext; + + fn count(&self) -> u64 { + self.imp.count() + } + fn iter(&self) -> impl Iterator + '_ { + self.imp.iter() + } + + type InsertCallbackId = TxnBucketInsertCallbackId; + + fn on_insert( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> TxnBucketInsertCallbackId { + TxnBucketInsertCallbackId(self.imp.on_insert(Box::new(callback))) + } + + fn remove_on_insert(&self, callback: TxnBucketInsertCallbackId) { + self.imp.remove_on_insert(callback.0) + } + + type DeleteCallbackId = TxnBucketDeleteCallbackId; + + fn on_delete( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> TxnBucketDeleteCallbackId { + TxnBucketDeleteCallbackId(self.imp.on_delete(Box::new(callback))) + } + + fn remove_on_delete(&self, callback: TxnBucketDeleteCallbackId) { + self.imp.remove_on_delete(callback.0) + } +} + +pub struct TxnBucketUpdateCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::TableWithPrimaryKey for TxnBucketTableHandle<'ctx> { + type UpdateCallbackId = TxnBucketUpdateCallbackId; + + fn on_update( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row, &Self::Row) + Send + 'static, + ) -> TxnBucketUpdateCallbackId { + TxnBucketUpdateCallbackId(self.imp.on_update(Box::new(callback))) + } + + fn remove_on_update(&self, callback: TxnBucketUpdateCallbackId) { + self.imp.remove_on_update(callback.0) + } +} + +/// Access to the `bucket_start_ms` unique index on the table `txn_bucket`, +/// which allows point queries on the field of the same name +/// via the [`TxnBucketBucketStartMsUnique::find`] method. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.txn_bucket().bucket_start_ms().find(...)`. +pub struct TxnBucketBucketStartMsUnique<'ctx> { + imp: __sdk::UniqueConstraintHandle, + phantom: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +impl<'ctx> TxnBucketTableHandle<'ctx> { + /// Get a handle on the `bucket_start_ms` unique index on the table `txn_bucket`. + pub fn bucket_start_ms(&self) -> TxnBucketBucketStartMsUnique<'ctx> { + TxnBucketBucketStartMsUnique { + imp: self.imp.get_unique_constraint::("bucket_start_ms"), + phantom: std::marker::PhantomData, + } + } +} + +impl<'ctx> TxnBucketBucketStartMsUnique<'ctx> { + /// Find the subscribed row whose `bucket_start_ms` column value is equal to `col_val`, + /// if such a row is present in the client cache. + pub fn find(&self, col_val: &u64) -> Option { + self.imp.find(col_val) + } +} + +#[doc(hidden)] +pub(super) fn register_table(client_cache: &mut __sdk::ClientCache) { + let _table = client_cache.get_or_make_table::("txn_bucket"); + _table.add_unique_constraint::("bucket_start_ms", |row| &row.bucket_start_ms); +} + +#[doc(hidden)] +pub(super) fn parse_table_update(raw_updates: __ws::v2::TableUpdate) -> __sdk::Result<__sdk::TableUpdate> { + __sdk::TableUpdate::parse_table_update(raw_updates).map_err(|e| { + __sdk::InternalError::failed_parse("TableUpdate", "TableUpdate") + .with_cause(e) + .into() + }) +} + +#[allow(non_camel_case_types)] +/// Extension trait for query builder access to the table `TxnBucket`. +/// +/// Implemented for [`__sdk::QueryTableAccessor`]. +pub trait txn_bucketQueryTableAccess { + #[allow(non_snake_case)] + /// Get a query builder for the table `TxnBucket`. + fn txn_bucket(&self) -> __sdk::__query_builder::Table; +} + +impl txn_bucketQueryTableAccess for __sdk::QueryTableAccessor { + fn txn_bucket(&self) -> __sdk::__query_builder::Table { + __sdk::__query_builder::Table::new("txn_bucket") + } +} diff --git a/tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_type.rs b/tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_type.rs new file mode 100644 index 00000000000..774dafd2063 --- /dev/null +++ b/tools/tpcc-runner/src/metrics_module_bindings/txn_bucket_type.rs @@ -0,0 +1,52 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct TxnBucket { + pub bucket_start_ms: u64, + pub count: u64, +} + +impl __sdk::InModule for TxnBucket { + type Module = super::RemoteModule; +} + +/// Column accessor struct for the table `TxnBucket`. +/// +/// Provides typed access to columns for query building. +pub struct TxnBucketCols { + pub bucket_start_ms: __sdk::__query_builder::Col, + pub count: __sdk::__query_builder::Col, +} + +impl __sdk::__query_builder::HasCols for TxnBucket { + type Cols = TxnBucketCols; + fn cols(table_name: &'static str) -> Self::Cols { + TxnBucketCols { + bucket_start_ms: __sdk::__query_builder::Col::new(table_name, "bucket_start_ms"), + count: __sdk::__query_builder::Col::new(table_name, "count"), + } + } +} + +/// Indexed column accessor struct for the table `TxnBucket`. +/// +/// Provides typed access to indexed columns for query building. +pub struct TxnBucketIxCols { + pub bucket_start_ms: __sdk::__query_builder::IxCol, +} + +impl __sdk::__query_builder::HasIxCols for TxnBucket { + type IxCols = TxnBucketIxCols; + fn ix_cols(table_name: &'static str) -> Self::IxCols { + TxnBucketIxCols { + bucket_start_ms: __sdk::__query_builder::IxCol::new(table_name, "bucket_start_ms"), + } + } +} + +impl __sdk::__query_builder::CanBeLookupTable for TxnBucket {} diff --git a/tools/tpcc-runner/src/metrics_module_bindings/txn_table.rs b/tools/tpcc-runner/src/metrics_module_bindings/txn_table.rs new file mode 100644 index 00000000000..8d867b3001d --- /dev/null +++ b/tools/tpcc-runner/src/metrics_module_bindings/txn_table.rs @@ -0,0 +1,151 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use super::txn_type::Txn; +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +/// Table handle for the table `txn`. +/// +/// Obtain a handle from the [`TxnTableAccess::txn`] method on [`super::RemoteTables`], +/// like `ctx.db.txn()`. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.txn().on_insert(...)`. +pub struct TxnTableHandle<'ctx> { + imp: __sdk::TableHandle, + ctx: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the table `txn`. +/// +/// Implemented for [`super::RemoteTables`]. +pub trait TxnTableAccess { + #[allow(non_snake_case)] + /// Obtain a [`TxnTableHandle`], which mediates access to the table `txn`. + fn txn(&self) -> TxnTableHandle<'_>; +} + +impl TxnTableAccess for super::RemoteTables { + fn txn(&self) -> TxnTableHandle<'_> { + TxnTableHandle { + imp: self.imp.get_table::("txn"), + ctx: std::marker::PhantomData, + } + } +} + +pub struct TxnInsertCallbackId(__sdk::CallbackId); +pub struct TxnDeleteCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::Table for TxnTableHandle<'ctx> { + type Row = Txn; + type EventContext = super::EventContext; + + fn count(&self) -> u64 { + self.imp.count() + } + fn iter(&self) -> impl Iterator + '_ { + self.imp.iter() + } + + type InsertCallbackId = TxnInsertCallbackId; + + fn on_insert(&self, callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static) -> TxnInsertCallbackId { + TxnInsertCallbackId(self.imp.on_insert(Box::new(callback))) + } + + fn remove_on_insert(&self, callback: TxnInsertCallbackId) { + self.imp.remove_on_insert(callback.0) + } + + type DeleteCallbackId = TxnDeleteCallbackId; + + fn on_delete(&self, callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static) -> TxnDeleteCallbackId { + TxnDeleteCallbackId(self.imp.on_delete(Box::new(callback))) + } + + fn remove_on_delete(&self, callback: TxnDeleteCallbackId) { + self.imp.remove_on_delete(callback.0) + } +} + +pub struct TxnUpdateCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::TableWithPrimaryKey for TxnTableHandle<'ctx> { + type UpdateCallbackId = TxnUpdateCallbackId; + + fn on_update( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row, &Self::Row) + Send + 'static, + ) -> TxnUpdateCallbackId { + TxnUpdateCallbackId(self.imp.on_update(Box::new(callback))) + } + + fn remove_on_update(&self, callback: TxnUpdateCallbackId) { + self.imp.remove_on_update(callback.0) + } +} + +/// Access to the `id` unique index on the table `txn`, +/// which allows point queries on the field of the same name +/// via the [`TxnIdUnique::find`] method. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.txn().id().find(...)`. +pub struct TxnIdUnique<'ctx> { + imp: __sdk::UniqueConstraintHandle, + phantom: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +impl<'ctx> TxnTableHandle<'ctx> { + /// Get a handle on the `id` unique index on the table `txn`. + pub fn id(&self) -> TxnIdUnique<'ctx> { + TxnIdUnique { + imp: self.imp.get_unique_constraint::("id"), + phantom: std::marker::PhantomData, + } + } +} + +impl<'ctx> TxnIdUnique<'ctx> { + /// Find the subscribed row whose `id` column value is equal to `col_val`, + /// if such a row is present in the client cache. + pub fn find(&self, col_val: &u64) -> Option { + self.imp.find(col_val) + } +} + +#[doc(hidden)] +pub(super) fn register_table(client_cache: &mut __sdk::ClientCache) { + let _table = client_cache.get_or_make_table::("txn"); + _table.add_unique_constraint::("id", |row| &row.id); +} + +#[doc(hidden)] +pub(super) fn parse_table_update(raw_updates: __ws::v2::TableUpdate) -> __sdk::Result<__sdk::TableUpdate> { + __sdk::TableUpdate::parse_table_update(raw_updates).map_err(|e| { + __sdk::InternalError::failed_parse("TableUpdate", "TableUpdate") + .with_cause(e) + .into() + }) +} + +#[allow(non_camel_case_types)] +/// Extension trait for query builder access to the table `Txn`. +/// +/// Implemented for [`__sdk::QueryTableAccessor`]. +pub trait txnQueryTableAccess { + #[allow(non_snake_case)] + /// Get a query builder for the table `Txn`. + fn txn(&self) -> __sdk::__query_builder::Table; +} + +impl txnQueryTableAccess for __sdk::QueryTableAccessor { + fn txn(&self) -> __sdk::__query_builder::Table { + __sdk::__query_builder::Table::new("txn") + } +} diff --git a/tools/tpcc-runner/src/metrics_module_bindings/txn_type.rs b/tools/tpcc-runner/src/metrics_module_bindings/txn_type.rs new file mode 100644 index 00000000000..f8a8ca08b78 --- /dev/null +++ b/tools/tpcc-runner/src/metrics_module_bindings/txn_type.rs @@ -0,0 +1,55 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct Txn { + pub id: u64, + pub measurement_time_ms: u64, + pub latency_ms: u16, +} + +impl __sdk::InModule for Txn { + type Module = super::RemoteModule; +} + +/// Column accessor struct for the table `Txn`. +/// +/// Provides typed access to columns for query building. +pub struct TxnCols { + pub id: __sdk::__query_builder::Col, + pub measurement_time_ms: __sdk::__query_builder::Col, + pub latency_ms: __sdk::__query_builder::Col, +} + +impl __sdk::__query_builder::HasCols for Txn { + type Cols = TxnCols; + fn cols(table_name: &'static str) -> Self::Cols { + TxnCols { + id: __sdk::__query_builder::Col::new(table_name, "id"), + measurement_time_ms: __sdk::__query_builder::Col::new(table_name, "measurement_time_ms"), + latency_ms: __sdk::__query_builder::Col::new(table_name, "latency_ms"), + } + } +} + +/// Indexed column accessor struct for the table `Txn`. +/// +/// Provides typed access to indexed columns for query building. +pub struct TxnIxCols { + pub id: __sdk::__query_builder::IxCol, +} + +impl __sdk::__query_builder::HasIxCols for Txn { + type IxCols = TxnIxCols; + fn ix_cols(table_name: &'static str) -> Self::IxCols { + TxnIxCols { + id: __sdk::__query_builder::IxCol::new(table_name, "id"), + } + } +} + +impl __sdk::__query_builder::CanBeLookupTable for Txn {} diff --git a/tools/tpcc-runner/src/module_bindings/tpcc_load_config_request_type.rs b/tools/tpcc-runner/src/module_bindings/tpcc_load_config_request_type.rs index 0e38d4f9edd..7f43a508490 100644 --- a/tools/tpcc-runner/src/module_bindings/tpcc_load_config_request_type.rs +++ b/tools/tpcc-runner/src/module_bindings/tpcc_load_config_request_type.rs @@ -10,6 +10,8 @@ pub struct TpccLoadConfigRequest { pub database_number: u32, pub num_databases: u32, pub warehouses_per_database: u32, + pub warehouse_id_offset: u32, + pub skip_items: bool, pub batch_size: u32, pub seed: u64, pub load_c_last: u32, From 5d441796e50b4b39e04f29a8a2adead246991a83 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Wed, 1 Apr 2026 09:03:44 -0700 Subject: [PATCH 32/33] Wait keying time before driver submits first request --- tools/tpcc-runner/src/driver.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/tpcc-runner/src/driver.rs b/tools/tpcc-runner/src/driver.rs index 0b7926ecfae..21e6d1c0f2b 100644 --- a/tools/tpcc-runner/src/driver.rs +++ b/tools/tpcc-runner/src/driver.rs @@ -233,6 +233,15 @@ async fn run_terminal(runtime: TerminalRuntime) -> Result<()> { } let kind = choose_transaction(&mut rng); + let keying_delay = keying_time(kind, config.keying_time_scale); + if !keying_delay.is_zero() && crate::summary::now_millis() < schedule.stop_ms { + tokio::time::sleep(keying_delay).await; + } + + if abort.load(Ordering::Relaxed) || crate::summary::now_millis() >= schedule.stop_ms { + break; + } + let started_ms = crate::summary::now_millis(); let context = TransactionContext { client: client.as_ref(), @@ -264,9 +273,9 @@ async fn run_terminal(runtime: TerminalRuntime) -> Result<()> { } } - let delay = keying_time(kind, config.keying_time_scale) + think_time(kind, config.think_time_scale, &mut rng); - if !delay.is_zero() && crate::summary::now_millis() < schedule.stop_ms { - tokio::time::sleep(delay).await; + let think_delay = think_time(kind, config.think_time_scale, &mut rng); + if !think_delay.is_zero() && crate::summary::now_millis() < schedule.stop_ms { + tokio::time::sleep(think_delay).await; } } Ok(()) From 8dd1528878ad4e7f5ed4288f9bc9c51c222dde12 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Wed, 1 Apr 2026 12:28:38 -0700 Subject: [PATCH 33/33] Stagger first request by tpcc terminals --- tools/tpcc-runner/src/driver.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/tpcc-runner/src/driver.rs b/tools/tpcc-runner/src/driver.rs index 21e6d1c0f2b..e0e0bf084ff 100644 --- a/tools/tpcc-runner/src/driver.rs +++ b/tools/tpcc-runner/src/driver.rs @@ -22,6 +22,8 @@ use crate::summary::{ use crate::topology::DatabaseTopology; use crate::tpcc::*; +const STARTUP_STAGGER_WINDOW_MS: u64 = 18_000; + struct TerminalRuntime { config: DriverConfig, client: Arc, @@ -226,6 +228,14 @@ async fn run_terminal(runtime: TerminalRuntime) -> Result<()> { ); } + let startup_stagger_ms = { + let mut startup_rng = rand::rng(); + startup_rng.random_range(0..=STARTUP_STAGGER_WINDOW_MS) + }; + if startup_stagger_ms > 0 && crate::summary::now_millis() < schedule.stop_ms { + tokio::time::sleep(Duration::from_millis(startup_stagger_ms)).await; + } + let mut rng = StdRng::seed_from_u64(seed); while !abort.load(Ordering::Relaxed) { if crate::summary::now_millis() >= schedule.stop_ms {