diff --git a/src/pipeline.jl b/src/pipeline.jl index 03ef17e5..b1ee5423 100644 --- a/src/pipeline.jl +++ b/src/pipeline.jl @@ -21,7 +21,6 @@ function _bm_sha()::String if git_source !== nothing for depot in Base.DEPOT_PATH clones_dir = joinpath(depot, "clones") - @show clones_dir isdir(clones_dir) || continue for clone in readdir(clones_dir; join=true) isdir(clone) || continue @@ -54,12 +53,12 @@ end # ── Per-model orchestrator ───────────────────────────────────────────────────── """ - test_model(omc, model, results_root, ref_root) → ModelResult + test_model(omc, model, results_root, ref_root; csv_max_size_mb) → ModelResult Run the four-phase pipeline for a single model and return its result. """ function test_model(omc::OMJulia.OMCSession, model::String, results_root::String, - ref_root::String)::ModelResult + ref_root::String; csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::ModelResult model_dir = joinpath(results_root, "files", model) mkpath(model_dir) @@ -78,7 +77,7 @@ function test_model(omc::OMJulia.OMCSession, model::String, results_root::String model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, "") # Phase 3 ────────────────────────────────────────────────────────────────── - sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model) + sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model; csv_max_size_mb) # Phase 4 (optional) ─────────────────────────────────────────────────────── cmp_total, cmp_pass, cmp_skip, cmp_csv = 0, 0, 0, "" @@ -112,13 +111,14 @@ Discovers models via OMC, runs `test_model` for each, then writes the HTML report. Returns a `Vector{ModelResult}`. """ function main(; - library :: String = LIBRARY, - version :: String = LIBRARY_VERSION, - filter :: Union{String,Nothing} = nothing, - omc_exe :: String = get(ENV, "OMC_EXE", "omc"), - results_root :: String = "", - ref_root :: String = get(ENV, "MAPLIB_REF", ""), - bm_options :: String = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"), + library :: String = LIBRARY, + version :: String = LIBRARY_VERSION, + filter :: Union{String,Nothing} = nothing, + omc_exe :: String = get(ENV, "OMC_EXE", "omc"), + results_root :: String = "", + ref_root :: String = get(ENV, "MAPLIB_REF", ""), + bm_options :: String = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"), + csv_max_size_mb :: Int = CSV_MAX_SIZE_MB, ) t0 = time() @@ -186,7 +186,7 @@ function main(; for (i, model) in enumerate(models) @info "[$i/$(length(models))] $model" - result = test_model(omc, model, results_root, ref_root) + result = test_model(omc, model, results_root, ref_root; csv_max_size_mb) push!(results, result) phase = result.sim_success ? "SIM OK" : @@ -221,7 +221,7 @@ function main(; time() - t0, ) - generate_report(results, results_root, info) + generate_report(results, results_root, info; csv_max_size_mb) write_summary(results, results_root, info) return results end diff --git a/src/report.jl b/src/report.jl index 5c48b4cc..63f81b32 100644 --- a/src/report.jl +++ b/src/report.jl @@ -14,7 +14,7 @@ function _status_cell(ok::Bool, t::Float64, logFile::Union{String,Nothing}) end """ - _cmp_cell(r, results_root) → HTML string + _cmp_cell(r, results_root, csv_max_size_mb) → HTML string Build the "Ref Cmp" table cell for one model row. @@ -25,20 +25,20 @@ Cell colour: - grey (`na`) — no reference data at all The sim CSV is always linked when the file exists (or shows "(CSV N/A)" when it -exceeded `CSV_MAX_SIZE_MB` and was replaced by a `.toobig` marker). When there +exceeded `csv_max_size_mb` MB and was replaced by a `.toobig` marker). When there are failures or skipped signals the detail page `_diff.html` — which holds zoomable charts and the variable-coverage table — is also linked. """ -function _cmp_cell(r::ModelResult, results_root::String) +function _cmp_cell(r::ModelResult, results_root::String, csv_max_size_mb::Int) short = split(r.name, ".")[end] # ── Sim CSV link ──────────────────────────────────────────────────────────── sim_csv = joinpath("files", r.name, "$(short)_sim.csv") abs_sim_csv = joinpath(results_root, sim_csv) - csv_link = if isfile(abs_sim_csv) + csv_link = if isfile(abs_sim_csv * ".toobig") + """ (CSV N/A)""" + elseif isfile(abs_sim_csv) """ (CSV)""" - elseif isfile(abs_sim_csv * ".toobig") - """ (CSV N/A)""" else "" end @@ -87,12 +87,12 @@ function _format_duration(t::Float64)::String end """ - generate_report(results, results_root, info) → report_path + generate_report(results, results_root, info; csv_max_size_mb) → report_path Write an `index.html` overview report to `results_root` and return its path. """ function generate_report(results::Vector{ModelResult}, results_root::String, - info::RunInfo) + info::RunInfo; csv_max_size_mb::Int = CSV_MAX_SIZE_MB) n = length(results) n_exp = count(r -> r.export_success, results) n_par = count(r -> r.parse_success, results) @@ -113,7 +113,7 @@ function generate_report(results::Vector{ModelResult}, results_root::String, $(_status_cell(r.export_success, r.export_time, rel_log_file_or_nothing(results_root, r.name, "export"))) $(_status_cell(r.parse_success, r.parse_time, rel_log_file_or_nothing(results_root, r.name, "parsing"))) $(_status_cell(r.sim_success, r.sim_time, rel_log_file_or_nothing(results_root, r.name, "sim"))) - $(_cmp_cell(r, results_root)) + $(_cmp_cell(r, results_root, csv_max_size_mb)) """ for r in results], "\n") bm_sha_link = isempty(info.bm_sha) ? "" : diff --git a/src/simulate.jl b/src/simulate.jl index 81cae84a..32936e98 100644 --- a/src/simulate.jl +++ b/src/simulate.jl @@ -6,15 +6,18 @@ import ModelingToolkit import Printf: @sprintf """ - run_simulate(ode_prob, model_dir, model) → (success, time, error, sol) + run_simulate(ode_prob, model_dir, model; csv_max_size_mb) → (success, time, error, sol) Solve `ode_prob` with Rodas5P (stiff solver). On success, also writes the full solution as a CSV file `_sim.csv` in `model_dir`. Writes a `_sim.log` file in `model_dir`. Returns `nothing` as the fourth element on failure. + +CSV files larger than `csv_max_size_mb` MiB are deleted and replaced with a +`_sim.csv.toobig` marker so that the report can note the omission. """ -function run_simulate(ode_prob, model_dir::String, - model::String)::Tuple{Bool,Float64,String,Any} +function run_simulate(ode_prob, model_dir::String, model::String; + csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any} sim_success = false sim_time = 0.0 sim_error = "" @@ -64,6 +67,13 @@ function run_simulate(ode_prob, model_dir::String, println(f, join(row, ",")) end end + csv_bytes = filesize(sim_csv) + if csv_bytes > csv_max_size_mb * 1024^2 + csv_mb = round(csv_bytes / 1024^2; digits=1) + @warn "Simulation CSV for $model is $(csv_mb) MB (> $(csv_max_size_mb) MB limit); skipping." + rm(sim_csv) + write(sim_csv * ".toobig", string(csv_bytes)) + end catch e @warn "Failed to write simulation CSV for $model: $(sprint(showerror, e))" end diff --git a/src/types.jl b/src/types.jl index 825b4621..b073c784 100644 --- a/src/types.jl +++ b/src/types.jl @@ -8,6 +8,10 @@ const LIBRARY_VERSION = "4.1.0" const CMP_REL_TOL = 0.02 const CMP_ABS_TOL = 1e-6 +# CSV files larger than this limit are not committed to gh-pages (GitHub +# enforces a 100 MB hard cap; we use a conservative 20 MB soft limit). +const CSV_MAX_SIZE_MB = 20 + # ── Comparison settings ──────────────────────────────────────────────────────── """