Skip sim CSVs exceeding a configurable size limit

src/pipeline.jl

@@ -21,7 +21,6 @@ function _bm_sha()::String
             if git_source !== nothing
                 for depot in Base.DEPOT_PATH
                     clones_dir = joinpath(depot, "clones")
-                    @show clones_dir
                     isdir(clones_dir) || continue
                     for clone in readdir(clones_dir; join=true)
                         isdir(clone) || continue
@@ -54,12 +53,12 @@ end
 # ── Per-model orchestrator ─────────────────────────────────────────────────────
 
 """
-    test_model(omc, model, results_root, ref_root) → ModelResult
+    test_model(omc, model, results_root, ref_root; csv_max_size_mb) → ModelResult
 
 Run the four-phase pipeline for a single model and return its result.
 """
 function test_model(omc::OMJulia.OMCSession, model::String, results_root::String,
-                    ref_root::String)::ModelResult
+                    ref_root::String; csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::ModelResult
     model_dir = joinpath(results_root, "files", model)
     mkpath(model_dir)
 
@@ -78,7 +77,7 @@ function test_model(omc::OMJulia.OMCSession, model::String, results_root::String
         model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, "")
 
     # Phase 3 ──────────────────────────────────────────────────────────────────
-    sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model)
+    sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model; csv_max_size_mb)
 
     # Phase 4 (optional) ───────────────────────────────────────────────────────
     cmp_total, cmp_pass, cmp_skip, cmp_csv = 0, 0, 0, ""
@@ -112,13 +111,14 @@ Discovers models via OMC, runs `test_model` for each, then writes the HTML
 report.  Returns a `Vector{ModelResult}`.
 """
 function main(;
-    library      :: String                = LIBRARY,
-    version      :: String                = LIBRARY_VERSION,
-    filter       :: Union{String,Nothing} = nothing,
-    omc_exe      :: String                = get(ENV, "OMC_EXE", "omc"),
-    results_root :: String                = "",
-    ref_root     :: String                = get(ENV, "MAPLIB_REF", ""),
-    bm_options   :: String                = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"),
+    library          :: String                = LIBRARY,
+    version          :: String                = LIBRARY_VERSION,
+    filter           :: Union{String,Nothing} = nothing,
+    omc_exe          :: String                = get(ENV, "OMC_EXE", "omc"),
+    results_root     :: String                = "",
+    ref_root         :: String                = get(ENV, "MAPLIB_REF", ""),
+    bm_options       :: String                = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"),
+    csv_max_size_mb  :: Int                   = CSV_MAX_SIZE_MB,
 )
     t0 = time()
 
@@ -186,7 +186,7 @@ function main(;
 
         for (i, model) in enumerate(models)
             @info "[$i/$(length(models))] $model"
-            result = test_model(omc, model, results_root, ref_root)
+            result = test_model(omc, model, results_root, ref_root; csv_max_size_mb)
             push!(results, result)
 
             phase = result.sim_success    ? "SIM OK"     :
@@ -221,7 +221,7 @@ function main(;
         time() - t0,
     )
 
-    generate_report(results, results_root, info)
+    generate_report(results, results_root, info; csv_max_size_mb)
     write_summary(results, results_root, info)
     return results
 end

src/report.jl

@@ -14,7 +14,7 @@ function _status_cell(ok::Bool, t::Float64, logFile::Union{String,Nothing})
 end
 
 """
-    _cmp_cell(r, results_root) → HTML string
+    _cmp_cell(r, results_root, csv_max_size_mb) → HTML string
 
 Build the "Ref Cmp" table cell for one model row.
 
@@ -25,20 +25,20 @@ Cell colour:
 - grey    (`na`)      — no reference data at all
 
 The sim CSV is always linked when the file exists (or shows "(CSV N/A)" when it
-exceeded `CSV_MAX_SIZE_MB` and was replaced by a `.toobig` marker).  When there
+exceeded `csv_max_size_mb` MB and was replaced by a `.toobig` marker).  When there
 are failures or skipped signals the detail page `<short>_diff.html` — which holds
 zoomable charts and the variable-coverage table — is also linked.
 """
-function _cmp_cell(r::ModelResult, results_root::String)
+function _cmp_cell(r::ModelResult, results_root::String, csv_max_size_mb::Int)
     short = split(r.name, ".")[end]
 
     # ── Sim CSV link ────────────────────────────────────────────────────────────
     sim_csv     = joinpath("files", r.name, "$(short)_sim.csv")
     abs_sim_csv = joinpath(results_root, sim_csv)
-    csv_link = if isfile(abs_sim_csv)
+    csv_link = if isfile(abs_sim_csv * ".toobig")
+        """ <span title="Result file exceeds $(csv_max_size_mb) MB and was not uploaded">(CSV N/A)</span>"""
+    elseif isfile(abs_sim_csv)
         """ <a href="$sim_csv">(CSV)</a>"""
-    elseif isfile(abs_sim_csv * ".toobig")
-        """ <span title="Result file exceeds $(CSV_MAX_SIZE_MB) MB and was not uploaded">(CSV N/A)</span>"""
     else
         ""
     end
@@ -87,12 +87,12 @@ function _format_duration(t::Float64)::String
 end
 
 """
-    generate_report(results, results_root, info) → report_path
+    generate_report(results, results_root, info; csv_max_size_mb) → report_path
 
 Write an `index.html` overview report to `results_root` and return its path.
 """
 function generate_report(results::Vector{ModelResult}, results_root::String,
-                         info::RunInfo)
+                         info::RunInfo; csv_max_size_mb::Int = CSV_MAX_SIZE_MB)
     n     = length(results)
     n_exp = count(r -> r.export_success, results)
     n_par = count(r -> r.parse_success,  results)
@@ -113,7 +113,7 @@ function generate_report(results::Vector{ModelResult}, results_root::String,
       $(_status_cell(r.export_success, r.export_time, rel_log_file_or_nothing(results_root, r.name, "export")))
       $(_status_cell(r.parse_success,  r.parse_time,  rel_log_file_or_nothing(results_root, r.name, "parsing")))
       $(_status_cell(r.sim_success,    r.sim_time,    rel_log_file_or_nothing(results_root, r.name, "sim")))
-      $(_cmp_cell(r, results_root))
+      $(_cmp_cell(r, results_root, csv_max_size_mb))
     </tr>""" for r in results], "\n")
 
     bm_sha_link = isempty(info.bm_sha) ? "" :

src/simulate.jl

@@ -6,15 +6,18 @@ import ModelingToolkit
 import Printf: @sprintf
 
 """
-    run_simulate(ode_prob, model_dir, model) → (success, time, error, sol)
+    run_simulate(ode_prob, model_dir, model; csv_max_size_mb) → (success, time, error, sol)
 
 Solve `ode_prob` with Rodas5P (stiff solver).  On success, also writes the
 full solution as a CSV file `<Short>_sim.csv` in `model_dir`.
 Writes a `<model>_sim.log` file in `model_dir`.
 Returns `nothing` as the fourth element on failure.
+
+CSV files larger than `csv_max_size_mb` MiB are deleted and replaced with a
+`<Short>_sim.csv.toobig` marker so that the report can note the omission.
 """
-function run_simulate(ode_prob, model_dir::String,
-                      model::String)::Tuple{Bool,Float64,String,Any}
+function run_simulate(ode_prob, model_dir::String, model::String;
+                      csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any}
     sim_success = false
     sim_time    = 0.0
     sim_error   = ""
@@ -64,6 +67,13 @@ function run_simulate(ode_prob, model_dir::String,
                     println(f, join(row, ","))
                 end
             end
+            csv_bytes = filesize(sim_csv)
+            if csv_bytes > csv_max_size_mb * 1024^2
+                csv_mb = round(csv_bytes / 1024^2; digits=1)
+                @warn "Simulation CSV for $model is $(csv_mb) MB (> $(csv_max_size_mb) MB limit); skipping."
+                rm(sim_csv)
+                write(sim_csv * ".toobig", string(csv_bytes))
+            end
         catch e
             @warn "Failed to write simulation CSV for $model: $(sprint(showerror, e))"
         end

src/types.jl

@@ -8,6 +8,10 @@ const LIBRARY_VERSION = "4.1.0"
 const CMP_REL_TOL = 0.02
 const CMP_ABS_TOL = 1e-6
 
+# CSV files larger than this limit are not committed to gh-pages (GitHub
+# enforces a 100 MB hard cap; we use a conservative 20 MB soft limit).
+const CSV_MAX_SIZE_MB = 20
+
 # ── Comparison settings ────────────────────────────────────────────────────────
 
 """