From 79d9ad32439768c3babe3c60ce4a5c1d56761450 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 May 2026 14:32:33 +0100 Subject: [PATCH 1/2] Initial implementation --- Project.toml | 12 +- README.md | 32 +++++ src/TimerOutputComparisons.jl | 237 +++++++++++++++++++++++++++++++++ src/TimerOutputsComparisons.jl | 5 - test/runtests.jl | 52 +++++++- 5 files changed, 328 insertions(+), 10 deletions(-) create mode 100644 src/TimerOutputComparisons.jl delete mode 100644 src/TimerOutputsComparisons.jl diff --git a/Project.toml b/Project.toml index b885dbe..7ee147a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,9 +1,17 @@ -name = "TimerOutputsComparisons" +name = "TimerOutputComparisons" uuid = "59c811e7-adb4-4b01-a05e-50a2dd0b99a2" +version = "1.0.0" authors = ["John Omotani and contributors"] -version = "1.0.0-DEV" + +[deps] +GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" +JLD = "4138dd39-2aa7-5051-a626-17a0bb65d9c8" +TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" [compat] +GLMakie = "0.13.10" +JLD = "0.13.5" +TimerOutputs = "0.5.29" julia = "1.10.10" [extras] diff --git a/README.md b/README.md index e83689f..79a36a3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,35 @@ # TimerOutputsComparisons [![Build Status](https://github.com/johnomotani/TimerOutputsComparisons.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/johnomotani/TimerOutputsComparisons.jl/actions/workflows/CI.yml?query=branch%3Amain) + +Provides some helper functions to save/load TimerOutput objects, and plot +comparisons of them. This may be useful to compare performance with different +settings, or between different versions of some code. + +In complex examples it may be useful to save the TimerOutput objects to files, +and then plot them as a separate post-processing step, so in the example below +we save and re-load the TimerOutput objects, even though it is possible to pass +them directly to `compare_timers()`. + +Usage +----- + +```julia +using TimerOutputComparisons +using TimerOutputs + +delay_times = [0.1, 0.2, 0.3] + +for dt ∈ delay_times + to = TimerOutput() + @timeit to "sleep" sleep(dt) + filename = "foo$dt.jld" + save_timer(filename, to) +end + +compare_timers(["foo$dt.jld" for dt ∈ delay_times]...) +``` + +To plot only one or two quantities, use the `include` kwarg and pass one or +more of `:ncalls`, `:time` and `:allocs` to `include`. For example +`compare_timers(["foo$dt.jld" for dt ∈ delay_times]...; include=:time)`. diff --git a/src/TimerOutputComparisons.jl b/src/TimerOutputComparisons.jl new file mode 100644 index 0000000..c2ad585 --- /dev/null +++ b/src/TimerOutputComparisons.jl @@ -0,0 +1,237 @@ +""" +TimerOutputComparisons +====================== + +Provides some helper functions to save/load TimerOutput objects, and plot comparisons of +them. This may be useful to compare performance with different settings, or between +different versions of some code. +""" +module TimerOutputComparisons + +export save_timer, load_timer, compare_timers +# Workaround for failures when JLD is not loaded in Main, see +# https://github.com/JuliaIO/JLD.jl/issues/252 +export JLD + +using GLMakie +using JLD +using TimerOutputs + +""" + save_timer(filename::AbstractString, to::TimerOutput, + timer_name::AbstractString="to") + +Save `to` to a JLD file called `filename`, in a variable called `timer_name`. `filename` +should end with ".jld". +""" +function save_timer(filename::AbstractString, to::TimerOutput, + timer_name::AbstractString="to") + if splitext(filename)[2] != ".jld" + error("`filename` should end in \".jld\" so that JLD format is used. Otherwise " + * "a TimerOutput might not be writable and re-loadable.") + end + JLD.save(filename, timer_name, to) +end + +""" + load_timer(filename::AbstractString, + timer_name::AbstractString="to")::TimerOutput + +Load a TimerOutput called `timer_name` from a JLD file called `filename`. +""" +function load_timer(filename::AbstractString, + timer_name::AbstractString="to")::TimerOutput + if splitext(filename)[2] != ".jld" + error("Expected `filename` to end in \".jld\" so that JLD format is used.") + end + to = JLD.load(filename, timer_name) + return to +end + +const possible_includes = (:ncalls, :time, :allocs) + +""" + compare_timers(timers::Union{AbstractString,Tuple{<:AbstractString,<:AbstractString},Tuple{TimerOutput,<:AbstractString}}...; + flatten=false, save_as=nothing, include=$possible_includes) + +Make a plot comparing `timers`. For `t` in `timers`: +* if `t` is an AbstractString, load a TimerOutput from the file named `t` using + `load_timer()`, and label it `t`. +* if `t` is a `Tuple{<:AbstractString,<:AbstractString}`, load a TimerOutput called `t[2]` + from the file named `t[1]` using `load_timer()`, and label it `t[1] * ":" * t[2]`. +* if `t` is a `Tuple{TimerOutput,<:AbstractString}`, use the TimerOutput `t[1]` and label + it `t[2]`. + +We assume that the TimerOutput objects in `timers` contain (mostly) the same timers, +otherwise this comparison will not make much sense. + +If `flatten=true`, the TimerOutput objects are flattened with `TimerOutputs.flatten()`. + +If a file-name is passed to `save_as` the plots are saved instead of being displayed +interactively. For example, `save_as="foo.png"` would result in the plots being saved as +"foo_ncalls.png", "foo_time.png", and "foo_allocs.png". + +To plot only one or two quantities, pass one or more of `:ncalls`, `:time` and `:allocs` +to `include`, for example `include=:time`. +""" +compare_timers + +function compare_timers(timers::Union{AbstractString,<:Tuple{AbstractString,AbstractString},<:Tuple{TimerOutput,AbstractString}}...; kwargs...) + function get_timer(t)::Tuple{TimerOutput,String} + if t isa Tuple{TimerOutput,<:AbstractString} + return (t[1], String(t[2])) + elseif t isa AbstractString + return (load_timer(t), splitext(String(t))[1]) + elseif t isa Tuple{<:AbstractString,<:AbstractString} + return (load_timer(t...), splitext(String(t[1]))[1] * ":" * String(t[2])) + else + error("Unsupported type $(typeof(t)) for t=$t.") + end + end + return compare_timers(Tuple(get_timer(t) for t ∈ timers)...; kwargs...) +end +function compare_timers(timers::Tuple{TimerOutput,String}...; + flatten=false, save_as=nothing, include=possible_includes) + + if isa(include, Symbol) + include = (include,) + end + for i ∈ include + if i ∉ possible_includes + error("'$i' is not a valid entry in include. Possible values are " + * "$possible_includes.") + end + end + if flatten + timers = map(t->(TimerOutputs.flatten(t[1]), t[2]), timers) + end + + to_list = [t[1] for t ∈ timers] + x_values = [t[2] for t ∈ timers] + + # Get names of all timers to plot. + timer_names = Vector{String}[] + function extract_names!(t, name) + if !isempty(name) && name ∉ timer_names + push!(timer_names, name) + end + inner_timers = t.inner_timers + if !isempty(inner_timers) + for k ∈ keys(inner_timers) + new_name = copy(name) + push!(new_name, k) + extract_names!(t[k], new_name) + end + end + return nothing + end + for t ∈ to_list + extract_names!(t, String[]) + end + + xticks = (1:length(x_values), x_values) + if :ncalls ∈ include + fig_ncalls = Figure() + ax_ncalls = Axis(fig_ncalls[1,1]; xticks=xticks, ylabel="ncalls") + else + fig_ncalls = nothing + ax_ncalls = nothing + end + if :time ∈ include + fig_time = Figure() + ax_time = Axis(fig_time[1,1]; xticks=xticks, ylabel="time (ms)") + else + fig_time = nothing + ax_time = nothing + end + if :allocs ∈ include + fig_allocs = Figure() + ax_allocs = Axis(fig_allocs[1,1]; xticks=xticks, ylabel="allocated (kB)") + else + fig_allocs = nothing + ax_allocs = nothing + end + + for name ∈ timer_names + plot_single_timer!(ax_ncalls, ax_time, ax_allocs, to_list, name, xticks) + end + + for (fig, ax) in zip((fig_ncalls, fig_time, fig_allocs), (ax_ncalls, ax_time, ax_allocs)) + if fig !== nothing + # Ensure the first row width is 3/4 of the column width so that the plot does not get + # squashed by the legend + rowsize!(fig.layout, 1, Aspect(1, 3/4)) + + Legend(fig[2,1], ax; tellwidth=false, tellheight=true) + + resize_to_layout!(fig) + end + end + + if save_as === nothing + backend = Makie.current_backend() + for fig in (fig_ncalls, fig_time, fig_allocs) + if fig !== nothing + DataInspector(fig) + display(backend.Screen(), fig) + end + end + else + prefix, suffix = splitext(save_as) + if fig_ncalls !== nothing + save(prefix * "_ncalls" * suffix, fig_ncalls) + end + if fig_time !== nothing + save(prefix * "_time" * suffix, fig_time) + end + if fig_allocs !== nothing + save(prefix * "_allocs" * suffix, fig_allocs) + end + end + + return fig_ncalls, fig_time, fig_allocs +end + +function get_single_timer(to::TimerOutput, name::Vector{String}) + for n ∈ name + if n ∈ keys(to.inner_timers) + to = to[n] + else + return nothing + end + end + return to +end + +function plot_single_timer!(ax_ncalls, ax_time, ax_allocs, to_list, name::Vector{String}, + xticks) + this_timer_list = [get_single_timer(to, name) for to ∈ to_list] + label = join(name, ":") + + xtick_values = xticks[2] + if ax_ncalls !== nothing + ncalls_values = [t === nothing ? NaN : TimerOutputs.ncalls(t) for t ∈ this_timer_list] + lines!(ax_ncalls, ncalls_values; + label, + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): ncalls=$(ncalls_values[round(Int64, p[1])])") + end + + # Convert times from ns to ms. + if ax_time !== nothing + time_values = [t === nothing ? NaN : TimerOutputs.time(t) * 1.0e-6 for t ∈ this_timer_list] + lines!(ax_time, time_values; + label, + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): time=$(time_values[round(Int64, p[1])]) ms") + end + + if ax_allocs !== nothing + allocs_values = [t === nothing ? NaN : TimerOutputs.allocated(t) / 1024 for t ∈ this_timer_list] + lines!(ax_allocs, allocs_values; + label, + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): allocs=$(allocs_values[round(Int64, p[1])]) kB") + end + + return nothing +end + +end diff --git a/src/TimerOutputsComparisons.jl b/src/TimerOutputsComparisons.jl deleted file mode 100644 index 7cd41af..0000000 --- a/src/TimerOutputsComparisons.jl +++ /dev/null @@ -1,5 +0,0 @@ -module TimerOutputsComparisons - -# Write your package code here. - -end diff --git a/test/runtests.jl b/test/runtests.jl index bb85a9d..10c4a6e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,52 @@ -using TimerOutputsComparisons +using TimerOutputComparisons +using TimerOutputs using Test -@testset "TimerOutputsComparisons.jl" begin - # Write your tests here. +function dump_timer(sleeptime, outputdir) + to = TimerOutput() + @timeit to "sleep" sleep(sleeptime) + filename = joinpath(outputdir, "foo$sleeptime.jld") + save_timer(filename, to) end +function dump_timer(sleeptime, outputdir, timer_name) + to = TimerOutput() + @timeit to "sleep" sleep(sleeptime) + filename = joinpath(outputdir, "foo$sleeptime.jld") + save_timer(filename, to, timer_name) +end + +function runtests() + @testset "TimerOutputComparisons.jl" begin + for flatten ∈ (false, true), include ∈ (nothing, :ncalls, :time, :allocs, + (:ncalls, :time), (:ncalls, :allocs), + (:time, :allocs), + (:allocs, :time, :ncalls)) + # This package mostly makes interactive plots, so this is primarily a smoke-test. + outputdir = tempname() + mkpath(outputdir) + + dump_timer(0.1, outputdir) + dump_timer(0.5, outputdir, "bar") + + to = TimerOutput() + @timeit to "sleep" sleep(1) + + @test isa(load_timer(joinpath(outputdir, "foo0.1.jld")), TimerOutput) + @test isa(load_timer(joinpath(outputdir, "foo0.5.jld"), "bar"), TimerOutput) + + if include === nothing + compare_timers(joinpath(outputdir, "foo0.1.jld"), + (joinpath(outputdir, "foo0.5.jld"), "bar"), + (to, "foo1"); + flatten, save_as=joinpath(outputdir, "foo.png")) + else + compare_timers(joinpath(outputdir, "foo0.1.jld"), + (joinpath(outputdir, "foo0.5.jld"), "bar"), + (to, "foo1"); + flatten, include, save_as=joinpath(outputdir, "foo.png")) + end + end + end +end + +runtests() From 2e78ae50c11b720d31938563c677981251efca39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 14:32:25 +0000 Subject: [PATCH 2/2] Bump julia-actions/setup-julia from 2 to 3 Bumps [julia-actions/setup-julia](https://github.com/julia-actions/setup-julia) from 2 to 3. - [Release notes](https://github.com/julia-actions/setup-julia/releases) - [Commits](https://github.com/julia-actions/setup-julia/compare/v2...v3) --- updated-dependencies: - dependency-name: julia-actions/setup-julia dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index aef66dd..7fc0c61 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -32,7 +32,7 @@ jobs: - x64 steps: - uses: actions/checkout@v6 - - uses: julia-actions/setup-julia@v2 + - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }}