From aa8856b8fd3396c7646e587cae2b6a519a426dcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Sat, 17 Jan 2026 15:21:21 +0100 Subject: [PATCH 1/7] collect large scenes FPS and usage --- src/bin/collect.rs | 49 +++++ src/metrics/large_scenes.rs | 345 ++++++++++++++++++++++++++++++++++++ src/metrics/mod.rs | 1 + 3 files changed, 395 insertions(+) create mode 100644 src/metrics/large_scenes.rs diff --git a/src/bin/collect.rs b/src/bin/collect.rs index 9ae3878ef..206e4e407 100644 --- a/src/bin/collect.rs +++ b/src/bin/collect.rs @@ -57,6 +57,14 @@ enum Commands { #[arg(short, long)] nb_frames: u32, }, + LargeScene { + #[arg(short, long)] + scene: String, + #[arg(short, long)] + parameters: String, + #[arg(short, long)] + nb_frames: u32, + }, Benchmarks, LlvmLines, All, @@ -231,6 +239,47 @@ impl Commands { ))] } } + Commands::LargeScene { + scene, + parameters, + nb_frames, + } => { + if scene.is_empty() { + vec![ + Box::new(stress_tests::StressTest::on( + "bistro".to_string(), + vec![], + 10000, + )), + Box::new(stress_tests::StressTest::on( + "caldera_hotel".to_string(), + vec![], + 10000, + )), + ] + } else { + let parameters: Vec = + parameters.split(' ').map(|s| s.to_string()).collect(); + let parameters = parameters + .chunks(2) + .filter(|p| p.len() == 2) + .map(|p| { + ( + p[0].clone(), + if p[1].is_empty() { + None + } else { + Some(p[1].clone()) + }, + ) + }) + .collect(); + + vec![Box::new(large_scenes::LargeScene::on( + scene, parameters, nb_frames, + ))] + } + } Commands::Benchmarks => { vec![Box::new(benchmarks::Benchmarks)] } diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs new file mode 100644 index 000000000..efc5cfe08 --- /dev/null +++ b/src/metrics/large_scenes.rs @@ -0,0 +1,345 @@ +use std::{ + collections::HashMap, + io::{BufRead, Write}, + path::{Path, PathBuf}, + thread, + time::{Duration, Instant}, +}; + +use crossbeam::channel::Receiver; +use xshell::{Shell, cmd}; + +use crate::Metrics; + +#[derive(Debug)] +pub struct LargeScene { + pub scene: String, + pub parameters: Vec<(String, Option)>, + pub nb_frames: u32, + pub features: Vec, +} + +impl LargeScene { + pub fn on(scene: String, parameters: Vec<(String, Option)>, nb_frames: u32) -> Self { + Self { + scene, + parameters, + nb_frames, + features: vec![], + } + } + + pub fn with_features(mut self, features: Vec<&str>) -> Self { + self.features = features.into_iter().map(|f| f.to_string()).collect(); + self + } +} + +impl Metrics for LargeScene { + fn prepare(&self) -> bool { + let scene = self.scene.clone(); + + if std::fs::copy( + format!("~/assets/{scene}"), + format!("examples/large_scenes/{scene}/assets"), + ) + .is_err() + { + return false; + } + + let sh = Shell::new().unwrap(); + let mut features = self.features.clone(); + features.push("bevy_ci_testing".to_string()); + let features = features + .into_iter() + .flat_map(|f| ["--features".to_string(), f]); + + cmd!(sh, "cargo build --release {features...} --package {scene}") + .run() + .is_ok() + } + + fn artifacts(&self) -> HashMap { + std::fs::File::create("done").unwrap(); + HashMap::from([( + format!( + "large-scene-fps.{}.{}", + self.scene, + self.parameters + .iter() + .map(|(p, v)| if let Some(v) = v { + format!("{}-{}", p, v) + } else { + p.clone() + }) + .fold("params".to_string(), |acc, s| format!("{}-{}", acc, s)) + ), + Path::new("done").to_path_buf(), + )]) + } + + fn collect(&self) -> HashMap { + let cpu = cpu_usage(); + let gpu = gpu_usage(); + + let key = format!( + "large-scene-fps.{}.{}", + self.scene, + self.parameters + .iter() + .map(|(p, v)| if let Some(v) = v { + format!("{}-{}", p, v) + } else { + p.clone() + }) + .fold("params".to_string(), |acc, s| format!("{}-{}", acc, s)) + ); + let config = "twitcher_config.ron"; + let mut config_file = std::fs::File::create(config).unwrap(); + config_file + .write_fmt(format_args!("(events: [({}, AppExit)])", self.nb_frames)) + .unwrap(); + let sh = Shell::new().unwrap(); + sh.set_var("CI_TESTING_CONFIG", config); + + let parameters = self + .parameters + .iter() + .flat_map(|(p, v)| { + if let Some(v) = v { + vec![format!("--{}", p), v.clone()] + } else { + vec![format!("--{}", p)] + } + }) + .collect::>(); + let scene = self.scene.clone(); + let mut features = self.features.clone(); + features.push("bevy_ci_testing".to_string()); + let features = features + .into_iter() + .flat_map(|f| ["--features".to_string(), f]); + + let cmd = cmd!( + sh, + "xvfb-run cargo run --release {features...} --package {scene} -- {parameters...}" + ); + let mut results = HashMap::new(); + + // Wait for the monitoring threads to start + let _ = cpu.recv(); + let _ = gpu.recv(); + // Clear channels + while cpu.try_recv().is_ok() {} + while gpu.try_recv().is_ok() {} + + let start = Instant::now(); + let Ok(output) = cmd.output() else { + // ignore failure due to a missing scene + return results; + }; + let elapsed = start.elapsed(); + + let cpu_usage = cpu.try_iter().skip(2).collect::>(); + while cpu.try_recv().is_ok() {} + std::mem::drop(cpu); + + let gpu_usage = gpu + .try_iter() + .filter(|v| v.sm != 0) + .skip(2) + .collect::>(); + while gpu.try_recv().is_ok() {} + std::mem::drop(gpu); + + let gpu_memory = gpu_usage.iter().map(|v| v.mem as f32).collect::>(); + let gpu_usage = gpu_usage.iter().map(|v| v.sm as f32).collect::>(); + + let fpss = output + .stdout + .lines() + .chain(output.stderr.lines()) + .map_while(|line| line.ok()) + .filter(|line| line.contains("fps")) + .filter(|line| line.contains("avg")) + .map(|line| line.split("fps").nth(1).unwrap().to_string()) + .map(|line| line.split("(").nth(0).unwrap().to_string()) + .map(|line| line.split(":").nth(1).unwrap().to_string()) + .map(|line| line.trim().to_string()) + .map(|line| line.parse::().unwrap()) + .collect::>(); + + if !fpss.is_empty() { + results.insert( + format!("{key}.mean"), + (statistical::mean(&fpss) * 1000.0) as u64, + ); + results.insert( + format!("{key}.median"), + (statistical::median(&fpss) * 1000.0) as u64, + ); + results.insert( + format!("{key}.min"), + fpss.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), + ); + results.insert( + format!("{key}.max"), + fpss.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), + ); + results.insert( + format!("{key}.std_dev"), + (statistical::standard_deviation(&fpss, None) * 1000.0) as u64, + ); + } + results.insert( + format!("{key}.cpu_usage.mean"), + (statistical::mean(&cpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.cpu_usage.median"), + (statistical::median(&cpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.cpu_usage.min"), + cpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), + ); + results.insert( + format!("{key}.cpu_usage.max"), + cpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), + ); + results.insert( + format!("{key}.cpu_usage.std_dev"), + (statistical::standard_deviation(&cpu_usage, None) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_usage.mean"), + (statistical::mean(&gpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_usage.median"), + (statistical::median(&gpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_usage.min"), + gpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), + ); + results.insert( + format!("{key}.gpu_usage.max"), + gpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), + ); + results.insert( + format!("{key}.gpu_usage.std_dev"), + (statistical::standard_deviation(&gpu_usage, None) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_memory.mean"), + (statistical::mean(&gpu_memory) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_memory.median"), + (statistical::median(&gpu_memory) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_memory.min"), + gpu_memory + .iter() + .map(|d| (d * 1000.0) as u64) + .min() + .unwrap(), + ); + results.insert( + format!("{key}.gpu_memory.max"), + gpu_memory + .iter() + .map(|d| (d * 1000.0) as u64) + .max() + .unwrap(), + ); + results.insert( + format!("{key}.gpu_memory.std_dev"), + (statistical::standard_deviation(&gpu_memory, None) * 1000.0) as u64, + ); + results.insert(format!("{key}.duration"), elapsed.as_millis() as u64); + results.insert(format!("{key}.frames"), self.nb_frames as u64); + + results + } +} + +fn cpu_usage() -> Receiver { + let (tx, rx) = crossbeam::channel::unbounded(); + + thread::spawn(move || { + let mut sys = sysinfo::System::new(); + let delay = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL.max(Duration::from_secs(1)); + + loop { + sys.refresh_cpu_usage(); + if tx.send(sys.global_cpu_usage()).is_err() { + break; + } + std::thread::sleep(delay); + } + }); + + rx +} + +#[derive(Debug)] +struct GpuUsage { + sm: u32, + mem: u32, +} + +fn gpu_usage() -> Receiver { + let (tx, rx) = crossbeam::channel::unbounded(); + use nvml_wrapper::{Nvml, error::NvmlError}; + + thread::spawn(move || { + let Ok(nvml) = Nvml::init() else { + println!("Couldn't load nvidia driver"); + return; + }; + let device = nvml.device_by_index(0).unwrap(); + let delay = Duration::from_secs(1); + + let mut timestamp = None; + + let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); + + loop { + let processes = match device.process_utilization_stats(timestamp) { + Ok(processes) => processes, + Err(NvmlError::NotFound) => { + // No process using the GPU found + if tx.send(GpuUsage { sm: 0, mem: 0 }).is_err() { + break; + } + continue; + } + Err(_) => { + println!("Couldn't get process utilization stats"); + break; + } + }; + + let process = &processes[0]; + timestamp = Some(process.timestamp); + + if tx + .send(GpuUsage { + sm: process.sm_util, + mem: process.mem_util, + }) + .is_err() + { + break; + } + std::thread::sleep(delay); + } + let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); + }); + + rx +} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index e704fa551..c6a53d923 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -2,6 +2,7 @@ pub mod benchmarks; pub mod binary_size; pub mod compile_time; pub mod crate_compile_time; +pub mod large_scenes; pub mod llvm_lines; pub mod stress_tests; pub mod wasm_binary_size; From 7ffa9cb1ff7a88366f31d818473faa18d94be017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Sat, 17 Jan 2026 15:49:51 +0100 Subject: [PATCH 2/7] copy directory --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/metrics/large_scenes.rs | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 23204bceb..29c15c906 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,6 +609,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -1939,6 +1945,7 @@ dependencies = [ "chrono", "clap", "crossbeam", + "fs_extra", "git2", "nvml-wrapper", "plotters", diff --git a/Cargo.toml b/Cargo.toml index ace0d7e10..2a537d8dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ crossbeam = "0.8.4" nvml-wrapper = "0.11.0" regex = "1.11" git2 = "0.20" +fs_extra = "1.3.0" diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs index efc5cfe08..52cfff20d 100644 --- a/src/metrics/large_scenes.rs +++ b/src/metrics/large_scenes.rs @@ -39,9 +39,10 @@ impl Metrics for LargeScene { fn prepare(&self) -> bool { let scene = self.scene.clone(); - if std::fs::copy( + if fs_extra::dir::copy( format!("~/assets/{scene}"), format!("examples/large_scenes/{scene}/assets"), + &fs_extra::dir::CopyOptions::new(), ) .is_err() { From 377e6b2f6797d8fd794c41dbc4e45a7353d25b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Sat, 17 Jan 2026 15:54:53 +0100 Subject: [PATCH 3/7] fix --- src/metrics/large_scenes.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs index 52cfff20d..2a11a8a44 100644 --- a/src/metrics/large_scenes.rs +++ b/src/metrics/large_scenes.rs @@ -42,7 +42,7 @@ impl Metrics for LargeScene { if fs_extra::dir::copy( format!("~/assets/{scene}"), format!("examples/large_scenes/{scene}/assets"), - &fs_extra::dir::CopyOptions::new(), + &fs_extra::dir::CopyOptions::new().copy_inside(true), ) .is_err() { @@ -51,7 +51,7 @@ impl Metrics for LargeScene { let sh = Shell::new().unwrap(); let mut features = self.features.clone(); - features.push("bevy_ci_testing".to_string()); + features.push("bevy/bevy_ci_testing".to_string()); let features = features .into_iter() .flat_map(|f| ["--features".to_string(), f]); @@ -117,7 +117,7 @@ impl Metrics for LargeScene { .collect::>(); let scene = self.scene.clone(); let mut features = self.features.clone(); - features.push("bevy_ci_testing".to_string()); + features.push("bevy/bevy_ci_testing".to_string()); let features = features .into_iter() .flat_map(|f| ["--features".to_string(), f]); From 0636b032f2b9c791d7e81f180f58efa72912e8e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Sat, 17 Jan 2026 16:01:07 +0100 Subject: [PATCH 4/7] better path --- src/metrics/large_scenes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs index 2a11a8a44..f8383a311 100644 --- a/src/metrics/large_scenes.rs +++ b/src/metrics/large_scenes.rs @@ -40,7 +40,7 @@ impl Metrics for LargeScene { let scene = self.scene.clone(); if fs_extra::dir::copy( - format!("~/assets/{scene}"), + format!("/assets/{scene}"), format!("examples/large_scenes/{scene}/assets"), &fs_extra::dir::CopyOptions::new().copy_inside(true), ) From f8794b93e8fde3439bbe7e6686617b2f4f853e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Sat, 17 Jan 2026 16:12:42 +0100 Subject: [PATCH 5/7] don't parse logs --- src/metrics/large_scenes.rs | 40 ++----------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs index f8383a311..09295a0ad 100644 --- a/src/metrics/large_scenes.rs +++ b/src/metrics/large_scenes.rs @@ -1,6 +1,6 @@ use std::{ collections::HashMap, - io::{BufRead, Write}, + io::Write, path::{Path, PathBuf}, thread, time::{Duration, Instant}, @@ -136,7 +136,7 @@ impl Metrics for LargeScene { while gpu.try_recv().is_ok() {} let start = Instant::now(); - let Ok(output) = cmd.output() else { + if cmd.output().is_err() { // ignore failure due to a missing scene return results; }; @@ -157,42 +157,6 @@ impl Metrics for LargeScene { let gpu_memory = gpu_usage.iter().map(|v| v.mem as f32).collect::>(); let gpu_usage = gpu_usage.iter().map(|v| v.sm as f32).collect::>(); - let fpss = output - .stdout - .lines() - .chain(output.stderr.lines()) - .map_while(|line| line.ok()) - .filter(|line| line.contains("fps")) - .filter(|line| line.contains("avg")) - .map(|line| line.split("fps").nth(1).unwrap().to_string()) - .map(|line| line.split("(").nth(0).unwrap().to_string()) - .map(|line| line.split(":").nth(1).unwrap().to_string()) - .map(|line| line.trim().to_string()) - .map(|line| line.parse::().unwrap()) - .collect::>(); - - if !fpss.is_empty() { - results.insert( - format!("{key}.mean"), - (statistical::mean(&fpss) * 1000.0) as u64, - ); - results.insert( - format!("{key}.median"), - (statistical::median(&fpss) * 1000.0) as u64, - ); - results.insert( - format!("{key}.min"), - fpss.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), - ); - results.insert( - format!("{key}.max"), - fpss.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), - ); - results.insert( - format!("{key}.std_dev"), - (statistical::standard_deviation(&fpss, None) * 1000.0) as u64, - ); - } results.insert( format!("{key}.cpu_usage.mean"), (statistical::mean(&cpu_usage) * 1000.0) as u64, From 14557c8ec7c54d28c1e53074bc14b9e83491b310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Sat, 17 Jan 2026 16:13:27 +0100 Subject: [PATCH 6/7] run --- src/metrics/large_scenes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs index 09295a0ad..fd0f3231e 100644 --- a/src/metrics/large_scenes.rs +++ b/src/metrics/large_scenes.rs @@ -136,7 +136,7 @@ impl Metrics for LargeScene { while gpu.try_recv().is_ok() {} let start = Instant::now(); - if cmd.output().is_err() { + if cmd.run().is_err() { // ignore failure due to a missing scene return results; }; From b76f478d9d99bca2bf964c7ae0e41e351b2d0f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Sat, 17 Jan 2026 16:40:08 +0100 Subject: [PATCH 7/7] more frames --- src/bin/collect.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/collect.rs b/src/bin/collect.rs index 206e4e407..20a985692 100644 --- a/src/bin/collect.rs +++ b/src/bin/collect.rs @@ -249,12 +249,12 @@ impl Commands { Box::new(stress_tests::StressTest::on( "bistro".to_string(), vec![], - 10000, + 25000, )), Box::new(stress_tests::StressTest::on( "caldera_hotel".to_string(), vec![], - 10000, + 25000, )), ] } else {