diff --git a/Cargo.lock b/Cargo.lock index 23204bceb..29c15c906 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,6 +609,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -1939,6 +1945,7 @@ dependencies = [ "chrono", "clap", "crossbeam", + "fs_extra", "git2", "nvml-wrapper", "plotters", diff --git a/Cargo.toml b/Cargo.toml index ace0d7e10..2a537d8dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ crossbeam = "0.8.4" nvml-wrapper = "0.11.0" regex = "1.11" git2 = "0.20" +fs_extra = "1.3.0" diff --git a/src/bin/collect.rs b/src/bin/collect.rs index 9ae3878ef..20a985692 100644 --- a/src/bin/collect.rs +++ b/src/bin/collect.rs @@ -57,6 +57,14 @@ enum Commands { #[arg(short, long)] nb_frames: u32, }, + LargeScene { + #[arg(short, long)] + scene: String, + #[arg(short, long)] + parameters: String, + #[arg(short, long)] + nb_frames: u32, + }, Benchmarks, LlvmLines, All, @@ -231,6 +239,47 @@ impl Commands { ))] } } + Commands::LargeScene { + scene, + parameters, + nb_frames, + } => { + if scene.is_empty() { + vec![ + Box::new(stress_tests::StressTest::on( + "bistro".to_string(), + vec![], + 25000, + )), + Box::new(stress_tests::StressTest::on( + "caldera_hotel".to_string(), + vec![], + 25000, + )), + ] + } else { + let parameters: Vec = + parameters.split(' ').map(|s| s.to_string()).collect(); + let parameters = parameters + .chunks(2) + .filter(|p| p.len() == 2) + .map(|p| { + ( + p[0].clone(), + if p[1].is_empty() { + None + } else { + Some(p[1].clone()) + }, + ) + }) + .collect(); + + vec![Box::new(large_scenes::LargeScene::on( + scene, parameters, nb_frames, + ))] + } + } Commands::Benchmarks => { vec![Box::new(benchmarks::Benchmarks)] } diff --git a/src/metrics/large_scenes.rs b/src/metrics/large_scenes.rs new file mode 100644 index 000000000..fd0f3231e --- /dev/null +++ b/src/metrics/large_scenes.rs @@ -0,0 +1,310 @@ +use std::{ + collections::HashMap, + io::Write, + path::{Path, PathBuf}, + thread, + time::{Duration, Instant}, +}; + +use crossbeam::channel::Receiver; +use xshell::{Shell, cmd}; + +use crate::Metrics; + +#[derive(Debug)] +pub struct LargeScene { + pub scene: String, + pub parameters: Vec<(String, Option)>, + pub nb_frames: u32, + pub features: Vec, +} + +impl LargeScene { + pub fn on(scene: String, parameters: Vec<(String, Option)>, nb_frames: u32) -> Self { + Self { + scene, + parameters, + nb_frames, + features: vec![], + } + } + + pub fn with_features(mut self, features: Vec<&str>) -> Self { + self.features = features.into_iter().map(|f| f.to_string()).collect(); + self + } +} + +impl Metrics for LargeScene { + fn prepare(&self) -> bool { + let scene = self.scene.clone(); + + if fs_extra::dir::copy( + format!("/assets/{scene}"), + format!("examples/large_scenes/{scene}/assets"), + &fs_extra::dir::CopyOptions::new().copy_inside(true), + ) + .is_err() + { + return false; + } + + let sh = Shell::new().unwrap(); + let mut features = self.features.clone(); + features.push("bevy/bevy_ci_testing".to_string()); + let features = features + .into_iter() + .flat_map(|f| ["--features".to_string(), f]); + + cmd!(sh, "cargo build --release {features...} --package {scene}") + .run() + .is_ok() + } + + fn artifacts(&self) -> HashMap { + std::fs::File::create("done").unwrap(); + HashMap::from([( + format!( + "large-scene-fps.{}.{}", + self.scene, + self.parameters + .iter() + .map(|(p, v)| if let Some(v) = v { + format!("{}-{}", p, v) + } else { + p.clone() + }) + .fold("params".to_string(), |acc, s| format!("{}-{}", acc, s)) + ), + Path::new("done").to_path_buf(), + )]) + } + + fn collect(&self) -> HashMap { + let cpu = cpu_usage(); + let gpu = gpu_usage(); + + let key = format!( + "large-scene-fps.{}.{}", + self.scene, + self.parameters + .iter() + .map(|(p, v)| if let Some(v) = v { + format!("{}-{}", p, v) + } else { + p.clone() + }) + .fold("params".to_string(), |acc, s| format!("{}-{}", acc, s)) + ); + let config = "twitcher_config.ron"; + let mut config_file = std::fs::File::create(config).unwrap(); + config_file + .write_fmt(format_args!("(events: [({}, AppExit)])", self.nb_frames)) + .unwrap(); + let sh = Shell::new().unwrap(); + sh.set_var("CI_TESTING_CONFIG", config); + + let parameters = self + .parameters + .iter() + .flat_map(|(p, v)| { + if let Some(v) = v { + vec![format!("--{}", p), v.clone()] + } else { + vec![format!("--{}", p)] + } + }) + .collect::>(); + let scene = self.scene.clone(); + let mut features = self.features.clone(); + features.push("bevy/bevy_ci_testing".to_string()); + let features = features + .into_iter() + .flat_map(|f| ["--features".to_string(), f]); + + let cmd = cmd!( + sh, + "xvfb-run cargo run --release {features...} --package {scene} -- {parameters...}" + ); + let mut results = HashMap::new(); + + // Wait for the monitoring threads to start + let _ = cpu.recv(); + let _ = gpu.recv(); + // Clear channels + while cpu.try_recv().is_ok() {} + while gpu.try_recv().is_ok() {} + + let start = Instant::now(); + if cmd.run().is_err() { + // ignore failure due to a missing scene + return results; + }; + let elapsed = start.elapsed(); + + let cpu_usage = cpu.try_iter().skip(2).collect::>(); + while cpu.try_recv().is_ok() {} + std::mem::drop(cpu); + + let gpu_usage = gpu + .try_iter() + .filter(|v| v.sm != 0) + .skip(2) + .collect::>(); + while gpu.try_recv().is_ok() {} + std::mem::drop(gpu); + + let gpu_memory = gpu_usage.iter().map(|v| v.mem as f32).collect::>(); + let gpu_usage = gpu_usage.iter().map(|v| v.sm as f32).collect::>(); + + results.insert( + format!("{key}.cpu_usage.mean"), + (statistical::mean(&cpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.cpu_usage.median"), + (statistical::median(&cpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.cpu_usage.min"), + cpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), + ); + results.insert( + format!("{key}.cpu_usage.max"), + cpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), + ); + results.insert( + format!("{key}.cpu_usage.std_dev"), + (statistical::standard_deviation(&cpu_usage, None) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_usage.mean"), + (statistical::mean(&gpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_usage.median"), + (statistical::median(&gpu_usage) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_usage.min"), + gpu_usage.iter().map(|d| (d * 1000.0) as u64).min().unwrap(), + ); + results.insert( + format!("{key}.gpu_usage.max"), + gpu_usage.iter().map(|d| (d * 1000.0) as u64).max().unwrap(), + ); + results.insert( + format!("{key}.gpu_usage.std_dev"), + (statistical::standard_deviation(&gpu_usage, None) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_memory.mean"), + (statistical::mean(&gpu_memory) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_memory.median"), + (statistical::median(&gpu_memory) * 1000.0) as u64, + ); + results.insert( + format!("{key}.gpu_memory.min"), + gpu_memory + .iter() + .map(|d| (d * 1000.0) as u64) + .min() + .unwrap(), + ); + results.insert( + format!("{key}.gpu_memory.max"), + gpu_memory + .iter() + .map(|d| (d * 1000.0) as u64) + .max() + .unwrap(), + ); + results.insert( + format!("{key}.gpu_memory.std_dev"), + (statistical::standard_deviation(&gpu_memory, None) * 1000.0) as u64, + ); + results.insert(format!("{key}.duration"), elapsed.as_millis() as u64); + results.insert(format!("{key}.frames"), self.nb_frames as u64); + + results + } +} + +fn cpu_usage() -> Receiver { + let (tx, rx) = crossbeam::channel::unbounded(); + + thread::spawn(move || { + let mut sys = sysinfo::System::new(); + let delay = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL.max(Duration::from_secs(1)); + + loop { + sys.refresh_cpu_usage(); + if tx.send(sys.global_cpu_usage()).is_err() { + break; + } + std::thread::sleep(delay); + } + }); + + rx +} + +#[derive(Debug)] +struct GpuUsage { + sm: u32, + mem: u32, +} + +fn gpu_usage() -> Receiver { + let (tx, rx) = crossbeam::channel::unbounded(); + use nvml_wrapper::{Nvml, error::NvmlError}; + + thread::spawn(move || { + let Ok(nvml) = Nvml::init() else { + println!("Couldn't load nvidia driver"); + return; + }; + let device = nvml.device_by_index(0).unwrap(); + let delay = Duration::from_secs(1); + + let mut timestamp = None; + + let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); + + loop { + let processes = match device.process_utilization_stats(timestamp) { + Ok(processes) => processes, + Err(NvmlError::NotFound) => { + // No process using the GPU found + if tx.send(GpuUsage { sm: 0, mem: 0 }).is_err() { + break; + } + continue; + } + Err(_) => { + println!("Couldn't get process utilization stats"); + break; + } + }; + + let process = &processes[0]; + timestamp = Some(process.timestamp); + + if tx + .send(GpuUsage { + sm: process.sm_util, + mem: process.mem_util, + }) + .is_err() + { + break; + } + std::thread::sleep(delay); + } + let _ = tx.try_send(GpuUsage { sm: 0, mem: 0 }); + }); + + rx +} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index e704fa551..c6a53d923 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -2,6 +2,7 @@ pub mod benchmarks; pub mod binary_size; pub mod compile_time; pub mod crate_compile_time; +pub mod large_scenes; pub mod llvm_lines; pub mod stress_tests; pub mod wasm_binary_size;