From f2156a8f482fb4211536ee4c6f4a024a853daadb Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Thu, 19 Feb 2026 15:11:25 +0200 Subject: [PATCH 01/18] refactor: migrate command line parsing to System.CommandLine --- src/BenchmarkDotNet/BenchmarkDotNet.csproj | 2 +- .../ConsoleArguments/CommandLineOptions.cs | 309 ++++++++---------- .../ConsoleArguments/ConfigParser.cs | 174 +++++++--- 3 files changed, 265 insertions(+), 220 deletions(-) diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index c4d8b56639..30ea1be724 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 259b2c7bdc..e420b8f350 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -1,18 +1,18 @@ +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.CommandLine; using System.IO; using System.Linq; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Configs; using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using CommandLine; -using CommandLine.Text; -using JetBrains.Annotations; using Perfolizer.Mathematics.OutlierDetection; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Toolchains.MonoAotLLVM; namespace BenchmarkDotNet.ConsoleArguments { @@ -20,251 +20,208 @@ namespace BenchmarkDotNet.ConsoleArguments [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] public class CommandLineOptions { + public static readonly Option BaseJobOption = new("--job", "-j") + { Description = "Dry/Short/Medium/Long or Default" }; + + public static readonly Option RuntimesOption = new("--runtimes", "-r") + { Description = "Target framework monikers" }; + + public static readonly Option ExportersOption = new("--exporters", "-e") + { Description = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML" }; + + public static readonly Option FiltersOption = new("--filter", "-f") + { Description = "Glob patterns" }; + + public static readonly Option MemoryOption = new("--memory", "-m") + { Description = "Prints memory statistics" }; + + public static readonly Option ArtifactsDirectoryOption = new("--artifacts", "-a") + { Description = "Path to artifacts directory" }; + + public static readonly Option RunInProcessOption = new("--inProcess", "-i") + { Description = "Run in Process" }; + + public static readonly Option ThreadingOption = new("--threading") + { Description = "Prints threading statistics" }; + + public static readonly Option ExceptionsOption = new("--exceptions") + { Description = "Prints exceptions statistics" }; + + public static readonly Option DisassemblyOption = new("--disassembly") + { Description = "Prints disassembly" }; + + public static readonly Option DisassemblerDepthOption = new("--disassemblerDepth") + { Description = "Disassembler recursive depth" }; + + public static readonly Option DisassemblerFiltersOption = new("--disassemblerFilters") + { Description = "Disassembler filters" }; + + public static readonly Option DisassemblerDiffOption = new("--disassemblerDiff") + { Description = "Disassembler diff" }; + + public static readonly Option ProfilerOption = new("--profiler") + { Description = "Profiler (ETW, EventPipe, etc)" }; + + public static readonly Option DisplayAllStatisticsOption = new("--allStats") + { Description = "Displays all statistics" }; + + public static readonly Option StatisticalTestThresholdOption = new("--statisticalTest") + { Description = "Statistical test threshold" }; + + public static readonly Option CoreRunPathsOption = new("--coreRun") + { Description = "Paths to CoreRun" }; + + public static readonly Option CliPathOption = new("--cli") + { Description = "Path to dotnet cli" }; + + public static readonly Option RestorePathOption = new("--restore") + { Description = "Path to restore packages" }; + + public static readonly Option ClrVersionOption = new("--clrVersion") + { Description = "CLR version" }; + + public static readonly Option JoinOption = new("--join") + { Description = "Join summary" }; + + public static readonly Option KeepBenchmarkFilesOption = new("--keepFiles") + { Description = "Keep benchmark files" }; + + public static readonly Option DontOverwriteResultsOption = new("--dontOverwrite") + { Description = "Don't overwrite results" }; + + public static readonly Option StopOnFirstErrorOption = new("--stopOnFirstError") + { Description = "Stop on first error" }; + + public static readonly Option DisableLogFileOption = new("--disableLogFile") + { Description = "Disable log file" }; + + public static readonly Option LogBuildOutputOption = new("--logBuildOutput") + { Description = "Log build output" }; + + public static readonly Option GenerateBinLogOption = new("--buildLog") + { Description = "Generate MSBuild bin log" }; + + public static readonly Option ApplesToApplesOption = new("--applesToApples") + { Description = "Apples to Apples" }; + + public static readonly Option ResumeOption = new("--resume") + { Description = "Resume" }; + + public static readonly Option TimeoutOption = new("--timeout") + { Description = "Timeout in seconds" }; + + public static readonly Option LaunchCountOption = new("--launchCount") + { Description = "Launch count" }; + + public static readonly Option WarmupCountOption = new("--warmupCount") + { Description = "Warmup iteration count" }; + + public static readonly Option IterationCountOption = new("--iterationCount") + { Description = "Iteration count" }; + + public static readonly Option InvocationCountOption = new("--invocationCount") + { Description = "Invocation count" }; + + public static readonly Option UnrollFactorOption = new("--unrollFactor") + { Description = "Unroll factor" }; + + public static readonly Option RunOnceOption = new("--runOnce") + { Description = "Run once per iteration" }; + + public static readonly Option MemoryRandomizationOption = new("--memoryRandomization") + { Description = "Memory randomization" }; + + public static readonly Option NoForcedGCsOption = new("--noForcedGCs") + { Description = "No forced GCs" }; + + public static readonly Option AllCategoriesOption = new("--allCategories") + { Description = "All Categories" }; + + public static readonly Option AnyCategoriesOption = new("--anyCategories") + { Description = "Any Categories" }; + + public static readonly Option AttributeNamesOption = new("--attributeNames") + { Description = "Attribute Names" }; + + public static readonly Option HiddenColumnsOption = new("--hiddenColumns") + { Description = "Hidden Columns" }; + private const int DefaultDisassemblerRecursiveDepth = 1; private bool useDisassemblyDiagnoser; - - [Option('j', "job", Required = false, Default = "Default", HelpText = "Dry/Short/Medium/Long or Default")] public string BaseJob { get; set; } = ""; - - [Option('r', "runtimes", Required = false, HelpText = "Full target framework moniker for .NET Core and .NET. For Mono just 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline!")] public IEnumerable Runtimes { get; set; } = []; - - [Option('e', "exporters", Required = false, HelpText = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML")] public IEnumerable Exporters { get; set; } = []; - - [Option('m', "memory", Required = false, Default = false, HelpText = "Prints memory statistics")] public bool UseMemoryDiagnoser { get; set; } - - [Option('t', "threading", Required = false, Default = false, HelpText = "Prints threading statistics")] public bool UseThreadingDiagnoser { get; set; } - - [Option("exceptions", Required = false, Default = false, HelpText = "Prints exception statistics")] public bool UseExceptionDiagnoser { get; set; } - - [Option('d', "disasm", Required = false, Default = false, HelpText = "Gets disassembly of benchmarked code")] public bool UseDisassemblyDiagnoser { get => useDisassemblyDiagnoser || DisassemblerRecursiveDepth != DefaultDisassemblerRecursiveDepth || DisassemblerFilters.Any(); set => useDisassemblyDiagnoser = value; } - - [Option('p', "profiler", Required = false, HelpText = "Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory")] public string? Profiler { get; set; } - - [Option('f', "filter", Required = false, HelpText = "Glob patterns")] public IEnumerable Filters { get; set; } = []; - - [Option('h', "hide", Required = false, HelpText = "Hides columns by name")] public IEnumerable HiddenColumns { get; set; } = []; - - [Option('i', "inProcess", Required = false, Default = false, HelpText = "Run benchmarks in Process")] public bool RunInProcess { get; set; } - - [Option('a', "artifacts", Required = false, HelpText = "Valid path to accessible directory")] public DirectoryInfo? ArtifactsDirectory { get; set; } - - [Option("outliers", Required = false, Default = OutlierMode.RemoveUpper, HelpText = "DontRemove/RemoveUpper/RemoveLower/RemoveAll")] public OutlierMode Outliers { get; set; } - - [Option("affinity", Required = false, HelpText = "Affinity mask to set for the benchmark process")] public int? Affinity { get; set; } - - [Option("allStats", Required = false, Default = false, HelpText = "Displays all statistics (min, max & more)")] public bool DisplayAllStatistics { get; set; } - - [Option("allCategories", Required = false, HelpText = "Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed")] public IEnumerable AllCategories { get; set; } = []; - - [Option("anyCategories", Required = false, HelpText = "Any Categories to run")] public IEnumerable AnyCategories { get; set; } = []; - - [Option("attribute", Required = false, HelpText = "Run all methods with given attribute (applied to class or method)")] public IEnumerable AttributeNames { get; set; } = []; - - [Option("join", Required = false, Default = false, HelpText = "Prints single table with results for all benchmarks")] public bool Join { get; set; } - - [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")] public bool KeepBenchmarkFiles { get; set; } - - [Option("noOverwrite", Required = false, Default = false, HelpText = "Determines if the exported result files should not be overwritten (be default they are overwritten).")] public bool DontOverwriteResults { get; set; } - - [Option("counters", Required = false, HelpText = "Hardware Counters", Separator = '+')] public IEnumerable HardwareCounters { get; set; } = []; - - [Option("cli", Required = false, HelpText = "Path to dotnet cli (optional).")] public FileInfo? CliPath { get; set; } - - [Option("packages", Required = false, HelpText = "The directory to restore packages to (optional).")] public DirectoryInfo? RestorePath { get; set; } - - [Option("coreRun", Required = false, HelpText = "Path(s) to CoreRun (optional).")] public IReadOnlyList CoreRunPaths { get; set; } = []; - - [Option("monoPath", Required = false, HelpText = "Optional path to Mono which should be used for running benchmarks.")] public FileInfo? MonoPath { get; set; } - - [Option("clrVersion", Required = false, HelpText = "Optional version of private CLR build used as the value of COMPLUS_Version env var.")] public string? ClrVersion { get; set; } - - [Option("ilCompilerVersion", Required = false, HelpText = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with NativeAOT. Example: \"7.0.0-preview.3.22123.2\"")] public string? ILCompilerVersion { get; set; } - - [Option("ilcPackages", Required = false, HelpText = @"Optional path to shipping packages produced by local dotnet/runtime build. Example: 'D:\projects\runtime\artifacts\packages\Release\Shipping\'")] public DirectoryInfo? IlcPackages { get; set; } - - [Option("launchCount", Required = false, HelpText = "How many times we should launch process with target benchmark. The default is 1.")] public int? LaunchCount { get; set; } - - [Option("warmupCount", Required = false, HelpText = "How many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic.")] public int? WarmupIterationCount { get; set; } - - [Option("minWarmupCount", Required = false, HelpText = "Minimum count of warmup iterations that should be performed. The default is 6.")] public int? MinWarmupIterationCount { get; set; } - - [Option("maxWarmupCount", Required = false, HelpText = "Maximum count of warmup iterations that should be performed. The default is 50.")] public int? MaxWarmupIterationCount { get; set; } - - [Option("iterationTime", Required = false, HelpText = "Desired time of execution of an iteration in milliseconds. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default")] public int? IterationTimeInMilliseconds { get; set; } - - [Option("iterationCount", Required = false, HelpText = "How many target iterations should be performed. By default calculated by the heuristic.")] public int? IterationCount { get; set; } - - [Option("minIterationCount", Required = false, HelpText = "Minimum number of iterations to run. The default is 15.")] public int? MinIterationCount { get; set; } - - [Option("maxIterationCount", Required = false, HelpText = "Maximum number of iterations to run. The default is 100.")] public int? MaxIterationCount { get; set; } - - [Option("invocationCount", Required = false, HelpText = "Invocation count in a single iteration. By default calculated by the heuristic.")] public long? InvocationCount { get; set; } - - [Option("unrollFactor", Required = false, HelpText = "How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default")] public int? UnrollFactor { get; set; } - - [Option("strategy", Required = false, HelpText = "The RunStrategy that should be used. Throughput/ColdStart/Monitoring.")] public RunStrategy? RunStrategy { get; set; } - - [Option("platform", Required = false, HelpText = "The Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64.")] public Platform? Platform { get; set; } - - [Option("runOncePerIteration", Required = false, Default = false, HelpText = "Run the benchmark exactly once per iteration.")] public bool RunOncePerIteration { get; set; } - - [Option("info", Required = false, Default = false, HelpText = "Print environment information.")] public bool PrintInformation { get; set; } - - [Option("apples", Required = false, Default = false, HelpText = "Runs apples-to-apples comparison for specified Jobs.")] public bool ApplesToApples { get; set; } - - [Option("list", Required = false, Default = ListBenchmarkCaseMode.Disabled, HelpText = "Prints all of the available benchmark names. Flat/Tree")] public ListBenchmarkCaseMode ListBenchmarkCaseMode { get; set; } - - [Option("disasmDepth", Required = false, Default = DefaultDisassemblerRecursiveDepth, HelpText = "Sets the recursive depth for the disassembler.")] public int DisassemblerRecursiveDepth { get; set; } - - [Option("disasmFilter", Required = false, HelpText = "Glob patterns applied to full method signatures by the the disassembler.")] public IEnumerable DisassemblerFilters { get; set; } = []; - - [Option("disasmDiff", Required = false, Default = false, HelpText = "Generates diff reports for the disassembler.")] public bool DisassemblerDiff { get; set; } - - [Option("logBuildOutput", Required = false, HelpText = "Log Build output.")] public bool LogBuildOutput { get; set; } - - [Option("generateBinLog", Required = false, HelpText = "Generate msbuild binlog for builds")] public bool GenerateMSBuildBinLog { get; set; } - - [Option("buildTimeout", Required = false, HelpText = "Build timeout in seconds.")] public int? TimeOutInSeconds { get; set; } - - [Option("wakeLock", Required = false, HelpText = "Prevents the system from entering sleep or turning off the display. None/System/Display.")] public WakeLockType? WakeLock { get; set; } - - [Option("stopOnFirstError", Required = false, Default = false, HelpText = "Stop on first error.")] public bool StopOnFirstError { get; set; } - - [Option("statisticalTest", Required = false, HelpText = "Threshold for Mann–Whitney U Test. Examples: 5%, 10ms, 100ns, 1s")] public string? StatisticalTestThreshold { get; set; } - - [Option("disableLogFile", Required = false, HelpText = "Disables the logfile.")] public bool DisableLogFile { get; set; } - - [Option("maxWidth", Required = false, HelpText = "Max parameter column width, the default is 20.")] public int? MaxParameterColumnWidth { get; set; } - - [Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")] public IEnumerable EnvironmentVariables { get; set; } = []; - - [Option("memoryRandomization", Required = false, HelpText = "Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration.")] public bool MemoryRandomization { get; set; } - - [Option("wasmEngine", Required = false, HelpText = "Full path to a java script engine used to run the benchmarks, used by Wasm toolchain.")] public FileInfo? WasmJavascriptEngine { get; set; } - - [Option("wasmArgs", Required = false, Default = "--expose_wasm", HelpText = "Arguments for the javascript engine used by Wasm toolchain.")] public string? WasmJavaScriptEngineArguments { get; set; } - - [Option("customRuntimePack", Required = false, HelpText = "Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently.")] public string? CustomRuntimePack { get; set; } - - [Option("AOTCompilerPath", Required = false, HelpText = "Path to Mono AOT compiler, used for MonoAotLLVM.")] public FileInfo? AOTCompilerPath { get; set; } - - [Option("AOTCompilerMode", Required = false, Default = MonoAotCompilerMode.mini, HelpText = "Mono AOT compiler mode, either 'mini' or 'llvm'")] public MonoAotCompilerMode AOTCompilerMode { get; set; } - - [Option("wasmDataDir", Required = false, HelpText = "Wasm data directory")] public DirectoryInfo? WasmDataDirectory { get; set; } - - [Option("wasmCoreCLR", Required = false, Default = false, HelpText = "Use CoreCLR runtime pack (Microsoft.NETCore.App.Runtime.browser-wasm) instead of the Mono runtime pack for WASM benchmarks.")] public bool WasmCoreCLR { get; set; } - - [Option("noForcedGCs", Required = false, HelpText = "Specifying would not forcefully induce any GCs.")] public bool NoForcedGCs { get; set; } - - [Option("noOverheadEvaluation", Required = false, HelpText = "Specifying would not run the evaluation overhead iterations.")] public bool NoEvaluationOverhead { get; set; } - - [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")] public bool Resume { get; set; } - internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any(); - [Usage(ApplicationAlias = "")] - [PublicAPI] - public static IEnumerable Examples - { - get - { - var shortName = new UnParserSettings { PreferShortName = true }; - var longName = new UnParserSettings { PreferShortName = false }; - - yield return new Example("Use Job.ShortRun for running the benchmarks", shortName, new CommandLineOptions { BaseJob = "short" }); - yield return new Example("Run benchmarks in process", shortName, new CommandLineOptions { RunInProcess = true }); - yield return new Example("Run benchmarks for .NET 4.7.2, .NET 8.0 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "net8.0", "Mono" } }); - yield return new Example("Run benchmarks for .NET Core 3.1, .NET 6.0 and .NET 8.0. .NET Core 3.1 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp3.1", "net6.0", "net8.0" } }); - yield return new Example("Use MemoryDiagnoser to get GC stats", shortName, new CommandLineOptions { UseMemoryDiagnoser = true }); - yield return new Example("Use DisassemblyDiagnoser to get disassembly", shortName, new CommandLineOptions { UseDisassemblyDiagnoser = true }); - yield return new Example("Use HardwareCountersDiagnoser to get hardware counter info", longName, new CommandLineOptions { HardwareCounters = new[] { nameof(HardwareCounter.CacheMisses), nameof(HardwareCounter.InstructionRetired) } }); - yield return new Example("Run all benchmarks exactly once", shortName, new CommandLineOptions { BaseJob = "Dry", Filters = new[] { Escape("*") } }); - yield return new Example("Run all benchmarks from System.Memory namespace", shortName, new CommandLineOptions { Filters = new[] { Escape("System.Memory*") } }); - yield return new Example("Run all benchmarks from ClassA and ClassB using type names", shortName, new CommandLineOptions { Filters = new[] { "ClassA", "ClassB" } }); - yield return new Example("Run all benchmarks from ClassA and ClassB using patterns", shortName, new CommandLineOptions { Filters = new[] { Escape("*.ClassA.*"), Escape("*.ClassB.*") } }); - yield return new Example("Run all benchmarks called `BenchmarkName` and show the results in single summary", longName, new CommandLineOptions { Join = true, Filters = new[] { Escape("*.BenchmarkName") } }); - yield return new Example("Run selected benchmarks once per iteration", longName, new CommandLineOptions { RunOncePerIteration = true }); - yield return new Example("Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations", longName, new CommandLineOptions { InvocationCount = 100, WarmupIterationCount = 1, IterationCount = 5}); - yield return new Example("Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations", longName, new CommandLineOptions { IterationTimeInMilliseconds = 250, MinIterationCount = 9, MaxIterationCount = 15}); - yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET Core 6.0 will be baseline because it was provided as first.", longName, - new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "net6.0", "net8.0" }, StatisticalTestThreshold = "5%" }); - yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, - new CommandLineOptions { EnvironmentVariables = new[] { "ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2" } }); - yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = new[] { "Mean", "Ratio" }, }); - } - } - private static string Escape(string input) => UserInteractionHelper.EscapeCommandExample(input); } } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index b9163654eb..689cb48e54 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -25,11 +25,14 @@ using BenchmarkDotNet.Toolchains.MonoAotLLVM; using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.NativeAot; -using CommandLine; using Perfolizer.Horology; using Perfolizer.Mathematics.OutlierDetection; using BenchmarkDotNet.Toolchains.Mono; using Perfolizer.Metrology; +using System.CommandLine; +using System.CommandLine.Parsing; +using System.Runtime.InteropServices; +using RuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; namespace BenchmarkDotNet.ConsoleArguments { @@ -73,24 +76,113 @@ public static class ConfigParser public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) { - (bool isSuccess, IConfig? config, CommandLineOptions? options) result = default; - var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); - if (!expandSuccess) - { - return (false, default, default); - } - + if (!expandSuccess) return (false, default, default); args = expandedArgs; - using (var parser = CreateParser(logger)) + + var rootCommand = new RootCommand("BenchmarkDotNet Command Line options"); + + rootCommand.Add(CommandLineOptions.BaseJobOption); + rootCommand.Add(CommandLineOptions.RuntimesOption); + rootCommand.Add(CommandLineOptions.ExportersOption); + rootCommand.Add(CommandLineOptions.FiltersOption); + rootCommand.Add(CommandLineOptions.AllCategoriesOption); + rootCommand.Add(CommandLineOptions.AnyCategoriesOption); + rootCommand.Add(CommandLineOptions.AttributeNamesOption); + rootCommand.Add(CommandLineOptions.HiddenColumnsOption); + rootCommand.Add(CommandLineOptions.MemoryOption); + rootCommand.Add(CommandLineOptions.ThreadingOption); + rootCommand.Add(CommandLineOptions.ExceptionsOption); + rootCommand.Add(CommandLineOptions.DisassemblyOption); + rootCommand.Add(CommandLineOptions.DisassemblerDepthOption); + rootCommand.Add(CommandLineOptions.DisassemblerFiltersOption); + rootCommand.Add(CommandLineOptions.DisassemblerDiffOption); + rootCommand.Add(CommandLineOptions.ProfilerOption); + rootCommand.Add(CommandLineOptions.DisplayAllStatisticsOption); + rootCommand.Add(CommandLineOptions.StatisticalTestThresholdOption); + rootCommand.Add(CommandLineOptions.CoreRunPathsOption); + rootCommand.Add(CommandLineOptions.CliPathOption); + rootCommand.Add(CommandLineOptions.RestorePathOption); + rootCommand.Add(CommandLineOptions.ArtifactsDirectoryOption); + rootCommand.Add(CommandLineOptions.ClrVersionOption); + rootCommand.Add(CommandLineOptions.RunInProcessOption); + rootCommand.Add(CommandLineOptions.JoinOption); + rootCommand.Add(CommandLineOptions.KeepBenchmarkFilesOption); + rootCommand.Add(CommandLineOptions.DontOverwriteResultsOption); + rootCommand.Add(CommandLineOptions.StopOnFirstErrorOption); + rootCommand.Add(CommandLineOptions.DisableLogFileOption); + rootCommand.Add(CommandLineOptions.LogBuildOutputOption); + rootCommand.Add(CommandLineOptions.GenerateBinLogOption); + rootCommand.Add(CommandLineOptions.ApplesToApplesOption); + rootCommand.Add(CommandLineOptions.ResumeOption); + rootCommand.Add(CommandLineOptions.TimeoutOption); + rootCommand.Add(CommandLineOptions.LaunchCountOption); + rootCommand.Add(CommandLineOptions.WarmupCountOption); + rootCommand.Add(CommandLineOptions.IterationCountOption); + rootCommand.Add(CommandLineOptions.InvocationCountOption); + rootCommand.Add(CommandLineOptions.UnrollFactorOption); + rootCommand.Add(CommandLineOptions.RunOnceOption); + rootCommand.Add(CommandLineOptions.MemoryRandomizationOption); + rootCommand.Add(CommandLineOptions.NoForcedGCsOption); + + var parseResult = rootCommand.Parse(args); + + var options = new CommandLineOptions { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig, args), options) : (false, default, default)) - .WithNotParsed(errors => result = (false, default, default)); - } + BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", + Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? Array.Empty(), + Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? Array.Empty(), + Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? Array.Empty(), + AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? Array.Empty(), + AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? Array.Empty(), + AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? Array.Empty(), + HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? Array.Empty(), + + UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), + UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), + UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), + UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), + DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), + DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? Array.Empty(), + DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", + DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), + StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", + + CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? Array.Empty(), + CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), + RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null + ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) + : null, + ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), + ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", + + RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), + Join = parseResult.GetValue(CommandLineOptions.JoinOption), + KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), + DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), + StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), + DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), + LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), + GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), + ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), + Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), + TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), + + LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), + WarmupIterationCount = parseResult.GetValue(CommandLineOptions.WarmupCountOption), + IterationCount = parseResult.GetValue(CommandLineOptions.IterationCountOption), + InvocationCount = parseResult.GetValue(CommandLineOptions.InvocationCountOption), + UnrollFactor = parseResult.GetValue(CommandLineOptions.UnrollFactorOption), + RunOncePerIteration = parseResult.GetValue(CommandLineOptions.RunOnceOption), + MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), + NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption) + }; - return result; + bool isSuccess = Validate(options, logger); + return isSuccess + ? (true, CreateConfig(options, globalConfig, args), options) + : (false, default, default); } private static (bool Success, string[] ExpandedTokens) ExpandResponseFile(string[] args, ILogger logger) @@ -193,42 +285,38 @@ string GetToken() return result; } } - internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) { - (bool isSuccess, CommandLineOptions? options) result = default; + var rootCommand = new RootCommand("BenchmarkDotNet Command Line options"); + rootCommand.Add(CommandLineOptions.BaseJobOption); - ILogger logger = NullLogger.Instance; - using (var parser = CreateParser(logger)) + var parseResult = rootCommand.Parse(args); + var options = new CommandLineOptions { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, options) : (false, default)) - .WithNotParsed(errors => result = (false, default)); - - if (!result.isSuccess) - { - updatedArgs = null; - return false; - } - - updater(result.options!); + BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "" + }; - updatedArgs = parser.FormatCommandLine(result.options, settings => settings.SkipDefault = true).Split(); - return true; + if (!Validate(options, NullLogger.Instance)) + { + updatedArgs = null; + return false; } + + updater(options); + updatedArgs = args; + return true; } - private static Parser CreateParser(ILogger logger) - => new Parser(settings => - { - settings.CaseInsensitiveEnumValues = true; - settings.CaseSensitive = false; - settings.EnableDashDash = true; - settings.IgnoreUnknownArguments = false; - settings.HelpWriter = new LoggerWrapper(logger); - settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); - }); + // private static Parser CreateParser(ILogger logger) + // => new Parser(settings => + // { + // settings.CaseInsensitiveEnumValues = true; + // settings.CaseSensitive = false; + // settings.EnableDashDash = true; + // settings.IgnoreUnknownArguments = false; + // settings.HelpWriter = new LoggerWrapper(logger); + // settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); + // }); private static bool Validate(CommandLineOptions options, ILogger logger) { From 63a04320def7df78b51e56347b181122076941c6 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Thu, 19 Feb 2026 15:51:24 +0200 Subject: [PATCH 02/18] apply code review: remove commented out code --- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 689cb48e54..1d3876901d 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -307,17 +307,6 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act return true; } - // private static Parser CreateParser(ILogger logger) - // => new Parser(settings => - // { - // settings.CaseInsensitiveEnumValues = true; - // settings.CaseSensitive = false; - // settings.EnableDashDash = true; - // settings.IgnoreUnknownArguments = false; - // settings.HelpWriter = new LoggerWrapper(logger); - // settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); - // }); - private static bool Validate(CommandLineOptions options, ILogger logger) { if (options.BaseJob.IsBlank() || !AvailableJobs.ContainsKey(options.BaseJob)) From da925e0548b7821135a7e4b8d49aa9d07a5ef569 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Thu, 19 Feb 2026 16:15:41 +0200 Subject: [PATCH 03/18] apply code review: port default values to System.CommandLine options --- src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index e420b8f350..0a1f2e4b07 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -21,7 +21,7 @@ namespace BenchmarkDotNet.ConsoleArguments public class CommandLineOptions { public static readonly Option BaseJobOption = new("--job", "-j") - { Description = "Dry/Short/Medium/Long or Default" }; + { Description = "Dry/Short/Medium/Long or Default", DefaultValueFactory = _ => "Default" }; public static readonly Option RuntimesOption = new("--runtimes", "-r") { Description = "Target framework monikers" }; @@ -51,7 +51,7 @@ public class CommandLineOptions { Description = "Prints disassembly" }; public static readonly Option DisassemblerDepthOption = new("--disassemblerDepth") - { Description = "Disassembler recursive depth" }; + { Description = "Disassembler recursive depth", DefaultValueFactory = _ => 1 }; public static readonly Option DisassemblerFiltersOption = new("--disassemblerFilters") { Description = "Disassembler filters" }; @@ -145,7 +145,7 @@ public class CommandLineOptions public static readonly Option HiddenColumnsOption = new("--hiddenColumns") { Description = "Hidden Columns" }; - + private const int DefaultDisassemblerRecursiveDepth = 1; private bool useDisassemblyDiagnoser; public string BaseJob { get; set; } = ""; From 66caf25787d66a6501eee5d14a6f13b3b44fc7ae Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Thu, 19 Feb 2026 21:16:24 +0200 Subject: [PATCH 04/18] fix: restore legacy option aliases, help texts, and fix --help invocation --- .../ConsoleArguments/CommandLineOptions.cs | 69 ++++++++++--------- .../ConsoleArguments/ConfigParser.cs | 11 +++ 2 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 0a1f2e4b07..ae1b18c004 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -24,7 +24,7 @@ public class CommandLineOptions { Description = "Dry/Short/Medium/Long or Default", DefaultValueFactory = _ => "Default" }; public static readonly Option RuntimesOption = new("--runtimes", "-r") - { Description = "Target framework monikers" }; + { Description = "Target framework monikers for .NET Core and .NET. For Mono tool 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline!" }; public static readonly Option ExportersOption = new("--exporters", "-e") { Description = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML" }; @@ -36,36 +36,42 @@ public class CommandLineOptions { Description = "Prints memory statistics" }; public static readonly Option ArtifactsDirectoryOption = new("--artifacts", "-a") - { Description = "Path to artifacts directory" }; + { Description = "Valid path to accessible directory" }; public static readonly Option RunInProcessOption = new("--inProcess", "-i") - { Description = "Run in Process" }; + { Description = "Run benchmarks in Process" }; - public static readonly Option ThreadingOption = new("--threading") + public static readonly Option ThreadingOption = new("--threading", "-t") { Description = "Prints threading statistics" }; public static readonly Option ExceptionsOption = new("--exceptions") { Description = "Prints exceptions statistics" }; - public static readonly Option DisassemblyOption = new("--disassembly") - { Description = "Prints disassembly" }; + public static readonly Option DisassemblyOption = new("--disasm", "-d") + { Description = "Gets disassembly of benchmarked code" }; - public static readonly Option DisassemblerDepthOption = new("--disassemblerDepth") - { Description = "Disassembler recursive depth", DefaultValueFactory = _ => 1 }; + public static readonly Option DisassemblerDepthOption = new("--disasmDepth") + { Description = "Sets the recursive depth for the disassembler. Default is 1.", DefaultValueFactory = _ => 1 }; - public static readonly Option DisassemblerFiltersOption = new("--disassemblerFilters") - { Description = "Disassembler filters" }; + public static readonly Option OutliersOption = new("--outliers") + { Description = "Specifies outlier detection mode. Default is DontRemove.", DefaultValueFactory = _ => OutlierMode.DontRemove }; - public static readonly Option DisassemblerDiffOption = new("--disassemblerDiff") - { Description = "Disassembler diff" }; + public static readonly Option WasmJavaScriptEngineArgumentsOption = new("--wasmArgs") + { Description = "Arguments for the javascript engine. The default is \"--expose_wasm\"", DefaultValueFactory = _ => "--expose_wasm" }; - public static readonly Option ProfilerOption = new("--profiler") - { Description = "Profiler (ETW, EventPipe, etc)" }; + public static readonly Option DisassemblerFiltersOption = new("--disasmFilter") + { Description = "Filters for the disassembler. Example: *Program*" }; + + public static readonly Option DisassemblerDiffOption = new("--disasmDiff") + { Description = "Generates diff reports for the disassembler." }; + + public static readonly Option ProfilerOption = new("--profiler", "-p") + { Description = "Profiles benchmarked code. Allowed values: [EP, ETW, CV]" }; public static readonly Option DisplayAllStatisticsOption = new("--allStats") - { Description = "Displays all statistics" }; + { Description = "Displays all statistics (min, max & more)" }; - public static readonly Option StatisticalTestThresholdOption = new("--statisticalTest") + public static readonly Option StatisticalTestThresholdOption = new("--statTest") { Description = "Statistical test threshold" }; public static readonly Option CoreRunPathsOption = new("--coreRun") @@ -81,10 +87,10 @@ public class CommandLineOptions { Description = "CLR version" }; public static readonly Option JoinOption = new("--join") - { Description = "Join summary" }; + { Description = "Prints single summary for all benchmarks" }; public static readonly Option KeepBenchmarkFilesOption = new("--keepFiles") - { Description = "Keep benchmark files" }; + { Description = "Determines if all auto-generated files should be kept or removed after running the benchmarks." }; public static readonly Option DontOverwriteResultsOption = new("--dontOverwrite") { Description = "Don't overwrite results" }; @@ -93,16 +99,16 @@ public class CommandLineOptions { Description = "Stop on first error" }; public static readonly Option DisableLogFileOption = new("--disableLogFile") - { Description = "Disable log file" }; + { Description = "Disables the log file" }; public static readonly Option LogBuildOutputOption = new("--logBuildOutput") - { Description = "Log build output" }; + { Description = "Logs build output" }; public static readonly Option GenerateBinLogOption = new("--buildLog") { Description = "Generate MSBuild bin log" }; public static readonly Option ApplesToApplesOption = new("--applesToApples") - { Description = "Apples to Apples" }; + { Description = "Apples to apples" }; public static readonly Option ResumeOption = new("--resume") { Description = "Resume" }; @@ -111,37 +117,37 @@ public class CommandLineOptions { Description = "Timeout in seconds" }; public static readonly Option LaunchCountOption = new("--launchCount") - { Description = "Launch count" }; + { Description = "How many times we should launch process with target benchmark. The default is 1." }; public static readonly Option WarmupCountOption = new("--warmupCount") - { Description = "Warmup iteration count" }; + { Description = "How many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic." }; public static readonly Option IterationCountOption = new("--iterationCount") - { Description = "Iteration count" }; + { Description = "How many target iterations should be performed. If you set it, the minIterationCount and maxIterationCount are ignored. By default calculated by the heuristic." }; public static readonly Option InvocationCountOption = new("--invocationCount") - { Description = "Invocation count" }; + { Description = "Invocation count in a single iteration. By default calculated by the heuristic." }; public static readonly Option UnrollFactorOption = new("--unrollFactor") - { Description = "Unroll factor" }; + { Description = "How many times the benchmark method will be invoked per one iteration of a generated loop." }; public static readonly Option RunOnceOption = new("--runOnce") - { Description = "Run once per iteration" }; + { Description = "Run the benchmark exactly once per iteration." }; public static readonly Option MemoryRandomizationOption = new("--memoryRandomization") { Description = "Memory randomization" }; public static readonly Option NoForcedGCsOption = new("--noForcedGCs") - { Description = "No forced GCs" }; + { Description = "Turns off forced garbage collections." }; public static readonly Option AllCategoriesOption = new("--allCategories") - { Description = "All Categories" }; + { Description = "Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed." }; public static readonly Option AnyCategoriesOption = new("--anyCategories") { Description = "Any Categories" }; - public static readonly Option AttributeNamesOption = new("--attributeNames") - { Description = "Attribute Names" }; + public static readonly Option AttributeNamesOption = new("--attribute") + { Description = "Run all methods with given attribute (applied to class or method)" }; public static readonly Option HiddenColumnsOption = new("--hiddenColumns") { Description = "Hidden Columns" }; @@ -165,6 +171,7 @@ public bool UseDisassemblyDiagnoser public bool RunInProcess { get; set; } public DirectoryInfo? ArtifactsDirectory { get; set; } public OutlierMode Outliers { get; set; } + public string? WasmArgs { get; set; } public int? Affinity { get; set; } public bool DisplayAllStatistics { get; set; } public IEnumerable AllCategories { get; set; } = []; diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 1d3876901d..ee83fbb1fe 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -32,6 +32,7 @@ using System.CommandLine; using System.CommandLine.Parsing; using System.Runtime.InteropServices; +using System.CommandLine.Invocation; using RuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; namespace BenchmarkDotNet.ConsoleArguments @@ -94,6 +95,8 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par rootCommand.Add(CommandLineOptions.ThreadingOption); rootCommand.Add(CommandLineOptions.ExceptionsOption); rootCommand.Add(CommandLineOptions.DisassemblyOption); + rootCommand.Add(CommandLineOptions.OutliersOption); + rootCommand.Add(CommandLineOptions.WasmJavaScriptEngineArgumentsOption); rootCommand.Add(CommandLineOptions.DisassemblerDepthOption); rootCommand.Add(CommandLineOptions.DisassemblerFiltersOption); rootCommand.Add(CommandLineOptions.DisassemblerDiffOption); @@ -127,6 +130,12 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par var parseResult = rootCommand.Parse(args); + if (parseResult.Errors.Any() || args.Any(a => a == "-h" || a == "--help" || a == "-?")) + { + parseResult.Invoke(); + return (false, default, default); + } + var options = new CommandLineOptions { BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", @@ -145,6 +154,8 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? Array.Empty(), DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), + WasmArgs = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", From bf962106878b3f4d1dd6d8a57aa4095ad9cbc948 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Fri, 20 Feb 2026 01:18:02 +0200 Subject: [PATCH 05/18] fix: set OutliersOption default to RemoveUpper and remove obsolete WasmJavascriptEngineArguments property --- src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs | 5 ++--- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index ae1b18c004..8a9c480786 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -54,7 +54,7 @@ public class CommandLineOptions { Description = "Sets the recursive depth for the disassembler. Default is 1.", DefaultValueFactory = _ => 1 }; public static readonly Option OutliersOption = new("--outliers") - { Description = "Specifies outlier detection mode. Default is DontRemove.", DefaultValueFactory = _ => OutlierMode.DontRemove }; + { Description = "Specifies outlier detection mode. Default is RemoveUpper.", DefaultValueFactory = _ => OutlierMode.RemoveUpper }; public static readonly Option WasmJavaScriptEngineArgumentsOption = new("--wasmArgs") { Description = "Arguments for the javascript engine. The default is \"--expose_wasm\"", DefaultValueFactory = _ => "--expose_wasm" }; @@ -171,7 +171,7 @@ public bool UseDisassemblyDiagnoser public bool RunInProcess { get; set; } public DirectoryInfo? ArtifactsDirectory { get; set; } public OutlierMode Outliers { get; set; } - public string? WasmArgs { get; set; } + public string? WasmJavaScriptEngineArguments { get; set; } public int? Affinity { get; set; } public bool DisplayAllStatistics { get; set; } public IEnumerable AllCategories { get; set; } = []; @@ -218,7 +218,6 @@ public bool UseDisassemblyDiagnoser public IEnumerable EnvironmentVariables { get; set; } = []; public bool MemoryRandomization { get; set; } public FileInfo? WasmJavascriptEngine { get; set; } - public string? WasmJavaScriptEngineArguments { get; set; } public string? CustomRuntimePack { get; set; } public FileInfo? AOTCompilerPath { get; set; } public MonoAotCompilerMode AOTCompilerMode { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index ee83fbb1fe..58f78eaa6c 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -155,7 +155,7 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? Array.Empty(), DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), - WasmArgs = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), + WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", From 152ecc825dbb5e42972a491324bacd741e6bb5be Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Fri, 20 Feb 2026 02:04:55 +0200 Subject: [PATCH 06/18] fix: add missing options and fix DashDash behavior --- .../ConsoleArguments/CommandLineOptions.cs | 216 ++++++++++++------ .../ConsoleArguments/ConfigParser.cs | 157 ++++++++----- 2 files changed, 247 insertions(+), 126 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 8a9c480786..84f1e72f27 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Configs; using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Helpers; using Perfolizer.Mathematics.OutlierDetection; using BenchmarkDotNet.Engines; @@ -20,101 +21,96 @@ namespace BenchmarkDotNet.ConsoleArguments [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] public class CommandLineOptions { + private const int DefaultDisassemblerRecursiveDepth = 1; + private bool useDisassemblyDiagnoser; + + // Options defined in the same order as the original CommandLineParser implementation public static readonly Option BaseJobOption = new("--job", "-j") { Description = "Dry/Short/Medium/Long or Default", DefaultValueFactory = _ => "Default" }; public static readonly Option RuntimesOption = new("--runtimes", "-r") - { Description = "Target framework monikers for .NET Core and .NET. For Mono tool 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline!" }; + { Description = "Full target framework moniker for .NET Core and .NET. For Mono just 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline!" }; public static readonly Option ExportersOption = new("--exporters", "-e") { Description = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML" }; - public static readonly Option FiltersOption = new("--filter", "-f") - { Description = "Glob patterns" }; - public static readonly Option MemoryOption = new("--memory", "-m") { Description = "Prints memory statistics" }; - public static readonly Option ArtifactsDirectoryOption = new("--artifacts", "-a") - { Description = "Valid path to accessible directory" }; - - public static readonly Option RunInProcessOption = new("--inProcess", "-i") - { Description = "Run benchmarks in Process" }; - public static readonly Option ThreadingOption = new("--threading", "-t") { Description = "Prints threading statistics" }; public static readonly Option ExceptionsOption = new("--exceptions") - { Description = "Prints exceptions statistics" }; + { Description = "Prints exception statistics" }; public static readonly Option DisassemblyOption = new("--disasm", "-d") { Description = "Gets disassembly of benchmarked code" }; - public static readonly Option DisassemblerDepthOption = new("--disasmDepth") - { Description = "Sets the recursive depth for the disassembler. Default is 1.", DefaultValueFactory = _ => 1 }; + public static readonly Option ProfilerOption = new("--profiler", "-p") + { Description = "Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory" }; - public static readonly Option OutliersOption = new("--outliers") - { Description = "Specifies outlier detection mode. Default is RemoveUpper.", DefaultValueFactory = _ => OutlierMode.RemoveUpper }; + public static readonly Option FiltersOption = new("--filter", "-f") + { Description = "Glob patterns" }; - public static readonly Option WasmJavaScriptEngineArgumentsOption = new("--wasmArgs") - { Description = "Arguments for the javascript engine. The default is \"--expose_wasm\"", DefaultValueFactory = _ => "--expose_wasm" }; + public static readonly Option HiddenColumnsOption = new("--hide", "-h") + { Description = "Hides columns by name" }; - public static readonly Option DisassemblerFiltersOption = new("--disasmFilter") - { Description = "Filters for the disassembler. Example: *Program*" }; + public static readonly Option RunInProcessOption = new("--inProcess", "-i") + { Description = "Run benchmarks in Process" }; - public static readonly Option DisassemblerDiffOption = new("--disasmDiff") - { Description = "Generates diff reports for the disassembler." }; + public static readonly Option ArtifactsDirectoryOption = new("--artifacts", "-a") + { Description = "Valid path to accessible directory" }; - public static readonly Option ProfilerOption = new("--profiler", "-p") - { Description = "Profiles benchmarked code. Allowed values: [EP, ETW, CV]" }; + public static readonly Option OutliersOption = new("--outliers") + { Description = "DontRemove/RemoveUpper/RemoveLower/RemoveAll", DefaultValueFactory = _ => OutlierMode.RemoveUpper }; + + public static readonly Option AffinityOption = new("--affinity") + { Description = "Affinity mask to set for the benchmark process" }; public static readonly Option DisplayAllStatisticsOption = new("--allStats") { Description = "Displays all statistics (min, max & more)" }; - public static readonly Option StatisticalTestThresholdOption = new("--statTest") - { Description = "Statistical test threshold" }; - - public static readonly Option CoreRunPathsOption = new("--coreRun") - { Description = "Paths to CoreRun" }; - - public static readonly Option CliPathOption = new("--cli") - { Description = "Path to dotnet cli" }; + public static readonly Option AllCategoriesOption = new("--allCategories") + { Description = "Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed" }; - public static readonly Option RestorePathOption = new("--restore") - { Description = "Path to restore packages" }; + public static readonly Option AnyCategoriesOption = new("--anyCategories") + { Description = "Any Categories to run" }; - public static readonly Option ClrVersionOption = new("--clrVersion") - { Description = "CLR version" }; + public static readonly Option AttributeNamesOption = new("--attribute") + { Description = "Run all methods with given attribute (applied to class or method)" }; public static readonly Option JoinOption = new("--join") - { Description = "Prints single summary for all benchmarks" }; + { Description = "Prints single table with results for all benchmarks" }; public static readonly Option KeepBenchmarkFilesOption = new("--keepFiles") { Description = "Determines if all auto-generated files should be kept or removed after running the benchmarks." }; - public static readonly Option DontOverwriteResultsOption = new("--dontOverwrite") - { Description = "Don't overwrite results" }; + public static readonly Option DontOverwriteResultsOption = new("--noOverwrite") + { Description = "Determines if the exported result files should not be overwritten (be default they are overwritten)." }; - public static readonly Option StopOnFirstErrorOption = new("--stopOnFirstError") - { Description = "Stop on first error" }; + public static readonly Option HardwareCountersOption = new("--counters") + { Description = "Hardware Counters" }; - public static readonly Option DisableLogFileOption = new("--disableLogFile") - { Description = "Disables the log file" }; + public static readonly Option CliPathOption = new("--cli") + { Description = "Path to dotnet cli (optional)." }; - public static readonly Option LogBuildOutputOption = new("--logBuildOutput") - { Description = "Logs build output" }; + public static readonly Option RestorePathOption = new("--packages") + { Description = "The directory to restore packages to (optional)." }; - public static readonly Option GenerateBinLogOption = new("--buildLog") - { Description = "Generate MSBuild bin log" }; + public static readonly Option CoreRunPathsOption = new("--coreRun") + { Description = "Path(s) to CoreRun (optional)." }; - public static readonly Option ApplesToApplesOption = new("--applesToApples") - { Description = "Apples to apples" }; + public static readonly Option MonoPathOption = new("--monoPath") + { Description = "Optional path to Mono which should be used for running benchmarks." }; - public static readonly Option ResumeOption = new("--resume") - { Description = "Resume" }; + public static readonly Option ClrVersionOption = new("--clrVersion") + { Description = "Optional version of private CLR build used as the value of COMPLUS_Version env var." }; - public static readonly Option TimeoutOption = new("--timeout") - { Description = "Timeout in seconds" }; + public static readonly Option ILCompilerVersionOption = new("--ilCompilerVersion") + { Description = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with NativeAOT. Example: \"7.0.0-preview.3.22123.2\"" }; + + public static readonly Option IlcPackagesOption = new("--ilcPackages") + { Description = "Optional path to shipping packages produced by local dotnet/runtime build." }; public static readonly Option LaunchCountOption = new("--launchCount") { Description = "How many times we should launch process with target benchmark. The default is 1." }; @@ -122,38 +118,118 @@ public class CommandLineOptions public static readonly Option WarmupCountOption = new("--warmupCount") { Description = "How many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic." }; + public static readonly Option MinWarmupCountOption = new("--minWarmupCount") + { Description = "Minimum count of warmup iterations that should be performed. The default is 6." }; + + public static readonly Option MaxWarmupCountOption = new("--maxWarmupCount") + { Description = "Maximum count of warmup iterations that should be performed. The default is 50." }; + + public static readonly Option IterationTimeOption = new("--iterationTime") + { Description = "Desired time of execution of an iteration in milliseconds. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default" }; + public static readonly Option IterationCountOption = new("--iterationCount") - { Description = "How many target iterations should be performed. If you set it, the minIterationCount and maxIterationCount are ignored. By default calculated by the heuristic." }; + { Description = "How many target iterations should be performed. By default calculated by the heuristic." }; + + public static readonly Option MinIterationCountOption = new("--minIterationCount") + { Description = "Minimum number of iterations to run. The default is 15." }; + + public static readonly Option MaxIterationCountOption = new("--maxIterationCount") + { Description = "Maximum number of iterations to run. The default is 100." }; public static readonly Option InvocationCountOption = new("--invocationCount") { Description = "Invocation count in a single iteration. By default calculated by the heuristic." }; public static readonly Option UnrollFactorOption = new("--unrollFactor") - { Description = "How many times the benchmark method will be invoked per one iteration of a generated loop." }; + { Description = "How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default" }; + + public static readonly Option RunStrategyOption = new("--strategy") + { Description = "The RunStrategy that should be used. Throughput/ColdStart/Monitoring." }; + + public static readonly Option PlatformOption = new("--platform") + { Description = "The Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64." }; - public static readonly Option RunOnceOption = new("--runOnce") + public static readonly Option RunOnceOption = new("--runOncePerIteration") { Description = "Run the benchmark exactly once per iteration." }; + public static readonly Option PrintInformationOption = new("--info") + { Description = "Print environment information." }; + + public static readonly Option ApplesToApplesOption = new("--apples") + { Description = "Runs apples-to-apples comparison for specified Jobs." }; + + public static readonly Option ListBenchmarkCaseModeOption = new("--list") + { Description = "Prints all of the available benchmark names. Flat/Tree", DefaultValueFactory = _ => ListBenchmarkCaseMode.Disabled }; + + public static readonly Option DisassemblerDepthOption = new("--disasmDepth") + { Description = "Sets the recursive depth for the disassembler.", DefaultValueFactory = _ => DefaultDisassemblerRecursiveDepth }; + + public static readonly Option DisassemblerFiltersOption = new("--disasmFilter") + { Description = "Glob patterns applied to full method signatures by the disassembler." }; + + public static readonly Option DisassemblerDiffOption = new("--disasmDiff") + { Description = "Generates diff reports for the disassembler." }; + + public static readonly Option LogBuildOutputOption = new("--logBuildOutput") + { Description = "Log Build output." }; + + public static readonly Option GenerateBinLogOption = new("--generateBinLog") + { Description = "Generate msbuild binlog for builds" }; + + public static readonly Option TimeoutOption = new("--buildTimeout") + { Description = "Build timeout in seconds." }; + + public static readonly Option WakeLockOption = new("--wakeLock") + { Description = "Prevents the system from entering sleep or turning off the display. None/System/Display." }; + + public static readonly Option StopOnFirstErrorOption = new("--stopOnFirstError") + { Description = "Stop on first error." }; + + public static readonly Option StatisticalTestThresholdOption = new("--statisticalTest") + { Description = "Threshold for Mann-Whitney U Test. Examples: 5%, 10ms, 100ns, 1s" }; + + public static readonly Option DisableLogFileOption = new("--disableLogFile") + { Description = "Disables the logfile." }; + + public static readonly Option MaxParameterColumnWidthOption = new("--maxWidth") + { Description = "Max parameter column width, the default is 20." }; + + public static readonly Option EnvironmentVariablesOption = new("--envVars") + { Description = "Colon separated environment variables (key:value)" }; + public static readonly Option MemoryRandomizationOption = new("--memoryRandomization") - { Description = "Memory randomization" }; + { Description = "Specifies whether Engine should allocate some random-sized memory between iterations." }; - public static readonly Option NoForcedGCsOption = new("--noForcedGCs") - { Description = "Turns off forced garbage collections." }; + public static readonly Option WasmJavascriptEngineOption = new("--wasmEngine") + { Description = "Full path to a java script engine used to run the benchmarks, used by Wasm toolchain." }; - public static readonly Option AllCategoriesOption = new("--allCategories") - { Description = "Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed." }; + public static readonly Option WasmJavaScriptEngineArgumentsOption = new("--wasmArgs") + { Description = "Arguments for the javascript engine used by Wasm toolchain.", DefaultValueFactory = _ => "--expose_wasm" }; - public static readonly Option AnyCategoriesOption = new("--anyCategories") - { Description = "Any Categories" }; + public static readonly Option CustomRuntimePackOption = new("--customRuntimePack") + { Description = "Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently." }; - public static readonly Option AttributeNamesOption = new("--attribute") - { Description = "Run all methods with given attribute (applied to class or method)" }; + public static readonly Option AOTCompilerPathOption = new("--AOTCompilerPath") + { Description = "Path to Mono AOT compiler, used for MonoAotLLVM." }; - public static readonly Option HiddenColumnsOption = new("--hiddenColumns") - { Description = "Hidden Columns" }; + public static readonly Option AOTCompilerModeOption = new("--AOTCompilerMode") + { Description = "Mono AOT compiler mode, either 'mini' or 'llvm'", DefaultValueFactory = _ => MonoAotCompilerMode.mini }; - private const int DefaultDisassemblerRecursiveDepth = 1; - private bool useDisassemblyDiagnoser; + public static readonly Option WasmDataDirectoryOption = new("--wasmDataDir") + { Description = "Wasm data directory" }; + + public static readonly Option WasmCoreCLROption = new("--wasmCoreCLR") + { Description = "Use CoreCLR runtime pack instead of the Mono runtime pack for WASM benchmarks." }; + + public static readonly Option NoForcedGCsOption = new("--noForcedGCs") + { Description = "Specifying would not forcefully induce any GCs." }; + + public static readonly Option NoEvaluationOverheadOption = new("--noOverheadEvaluation") + { Description = "Specifying would not run the evaluation overhead iterations." }; + + public static readonly Option ResumeOption = new("--resume") + { Description = "Continue the execution if the last run was stopped." }; + + // Properties public string BaseJob { get; set; } = ""; public IEnumerable Runtimes { get; set; } = []; public IEnumerable Exporters { get; set; } = []; @@ -171,7 +247,6 @@ public bool UseDisassemblyDiagnoser public bool RunInProcess { get; set; } public DirectoryInfo? ArtifactsDirectory { get; set; } public OutlierMode Outliers { get; set; } - public string? WasmJavaScriptEngineArguments { get; set; } public int? Affinity { get; set; } public bool DisplayAllStatistics { get; set; } public IEnumerable AllCategories { get; set; } = []; @@ -218,6 +293,7 @@ public bool UseDisassemblyDiagnoser public IEnumerable EnvironmentVariables { get; set; } = []; public bool MemoryRandomization { get; set; } public FileInfo? WasmJavascriptEngine { get; set; } + public string? WasmJavaScriptEngineArguments { get; set; } public string? CustomRuntimePack { get; set; } public FileInfo? AOTCompilerPath { get; set; } public MonoAotCompilerMode AOTCompilerMode { get; set; } @@ -230,4 +306,4 @@ public bool UseDisassemblyDiagnoser private static string Escape(string input) => UserInteractionHelper.EscapeCommandExample(input); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 58f78eaa6c..4a7c42d89c 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -82,55 +82,80 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par args = expandedArgs; var rootCommand = new RootCommand("BenchmarkDotNet Command Line options"); + rootCommand.TreatUnmatchedTokensAsErrors = false; rootCommand.Add(CommandLineOptions.BaseJobOption); rootCommand.Add(CommandLineOptions.RuntimesOption); rootCommand.Add(CommandLineOptions.ExportersOption); - rootCommand.Add(CommandLineOptions.FiltersOption); - rootCommand.Add(CommandLineOptions.AllCategoriesOption); - rootCommand.Add(CommandLineOptions.AnyCategoriesOption); - rootCommand.Add(CommandLineOptions.AttributeNamesOption); - rootCommand.Add(CommandLineOptions.HiddenColumnsOption); rootCommand.Add(CommandLineOptions.MemoryOption); rootCommand.Add(CommandLineOptions.ThreadingOption); rootCommand.Add(CommandLineOptions.ExceptionsOption); rootCommand.Add(CommandLineOptions.DisassemblyOption); - rootCommand.Add(CommandLineOptions.OutliersOption); - rootCommand.Add(CommandLineOptions.WasmJavaScriptEngineArgumentsOption); - rootCommand.Add(CommandLineOptions.DisassemblerDepthOption); - rootCommand.Add(CommandLineOptions.DisassemblerFiltersOption); - rootCommand.Add(CommandLineOptions.DisassemblerDiffOption); rootCommand.Add(CommandLineOptions.ProfilerOption); - rootCommand.Add(CommandLineOptions.DisplayAllStatisticsOption); - rootCommand.Add(CommandLineOptions.StatisticalTestThresholdOption); - rootCommand.Add(CommandLineOptions.CoreRunPathsOption); - rootCommand.Add(CommandLineOptions.CliPathOption); - rootCommand.Add(CommandLineOptions.RestorePathOption); - rootCommand.Add(CommandLineOptions.ArtifactsDirectoryOption); - rootCommand.Add(CommandLineOptions.ClrVersionOption); + rootCommand.Add(CommandLineOptions.FiltersOption); + rootCommand.Add(CommandLineOptions.HiddenColumnsOption); rootCommand.Add(CommandLineOptions.RunInProcessOption); + rootCommand.Add(CommandLineOptions.ArtifactsDirectoryOption); + rootCommand.Add(CommandLineOptions.OutliersOption); + rootCommand.Add(CommandLineOptions.AffinityOption); + rootCommand.Add(CommandLineOptions.DisplayAllStatisticsOption); + rootCommand.Add(CommandLineOptions.AllCategoriesOption); + rootCommand.Add(CommandLineOptions.AnyCategoriesOption); + rootCommand.Add(CommandLineOptions.AttributeNamesOption); rootCommand.Add(CommandLineOptions.JoinOption); rootCommand.Add(CommandLineOptions.KeepBenchmarkFilesOption); rootCommand.Add(CommandLineOptions.DontOverwriteResultsOption); - rootCommand.Add(CommandLineOptions.StopOnFirstErrorOption); - rootCommand.Add(CommandLineOptions.DisableLogFileOption); - rootCommand.Add(CommandLineOptions.LogBuildOutputOption); - rootCommand.Add(CommandLineOptions.GenerateBinLogOption); - rootCommand.Add(CommandLineOptions.ApplesToApplesOption); - rootCommand.Add(CommandLineOptions.ResumeOption); - rootCommand.Add(CommandLineOptions.TimeoutOption); + rootCommand.Add(CommandLineOptions.HardwareCountersOption); + rootCommand.Add(CommandLineOptions.CliPathOption); + rootCommand.Add(CommandLineOptions.RestorePathOption); + rootCommand.Add(CommandLineOptions.CoreRunPathsOption); + rootCommand.Add(CommandLineOptions.MonoPathOption); + rootCommand.Add(CommandLineOptions.ClrVersionOption); + rootCommand.Add(CommandLineOptions.ILCompilerVersionOption); + rootCommand.Add(CommandLineOptions.IlcPackagesOption); rootCommand.Add(CommandLineOptions.LaunchCountOption); rootCommand.Add(CommandLineOptions.WarmupCountOption); + rootCommand.Add(CommandLineOptions.MinWarmupCountOption); + rootCommand.Add(CommandLineOptions.MaxWarmupCountOption); + rootCommand.Add(CommandLineOptions.IterationTimeOption); rootCommand.Add(CommandLineOptions.IterationCountOption); + rootCommand.Add(CommandLineOptions.MinIterationCountOption); + rootCommand.Add(CommandLineOptions.MaxIterationCountOption); rootCommand.Add(CommandLineOptions.InvocationCountOption); rootCommand.Add(CommandLineOptions.UnrollFactorOption); + rootCommand.Add(CommandLineOptions.RunStrategyOption); + rootCommand.Add(CommandLineOptions.PlatformOption); rootCommand.Add(CommandLineOptions.RunOnceOption); + rootCommand.Add(CommandLineOptions.PrintInformationOption); + rootCommand.Add(CommandLineOptions.ApplesToApplesOption); + rootCommand.Add(CommandLineOptions.ListBenchmarkCaseModeOption); + rootCommand.Add(CommandLineOptions.DisassemblerDepthOption); + rootCommand.Add(CommandLineOptions.DisassemblerFiltersOption); + rootCommand.Add(CommandLineOptions.DisassemblerDiffOption); + rootCommand.Add(CommandLineOptions.LogBuildOutputOption); + rootCommand.Add(CommandLineOptions.GenerateBinLogOption); + rootCommand.Add(CommandLineOptions.TimeoutOption); + rootCommand.Add(CommandLineOptions.WakeLockOption); + rootCommand.Add(CommandLineOptions.StopOnFirstErrorOption); + rootCommand.Add(CommandLineOptions.StatisticalTestThresholdOption); + rootCommand.Add(CommandLineOptions.DisableLogFileOption); + rootCommand.Add(CommandLineOptions.MaxParameterColumnWidthOption); + rootCommand.Add(CommandLineOptions.EnvironmentVariablesOption); rootCommand.Add(CommandLineOptions.MemoryRandomizationOption); + rootCommand.Add(CommandLineOptions.WasmJavascriptEngineOption); + rootCommand.Add(CommandLineOptions.WasmJavaScriptEngineArgumentsOption); + rootCommand.Add(CommandLineOptions.CustomRuntimePackOption); + rootCommand.Add(CommandLineOptions.AOTCompilerPathOption); + rootCommand.Add(CommandLineOptions.AOTCompilerModeOption); + rootCommand.Add(CommandLineOptions.WasmDataDirectoryOption); + rootCommand.Add(CommandLineOptions.WasmCoreCLROption); rootCommand.Add(CommandLineOptions.NoForcedGCsOption); + rootCommand.Add(CommandLineOptions.NoEvaluationOverheadOption); + rootCommand.Add(CommandLineOptions.ResumeOption); var parseResult = rootCommand.Parse(args); - if (parseResult.Errors.Any() || args.Any(a => a == "-h" || a == "--help" || a == "-?")) + if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) { parseResult.Invoke(); return (false, default, default); @@ -141,53 +166,73 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? Array.Empty(), Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? Array.Empty(), - Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? Array.Empty(), - AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? Array.Empty(), - AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? Array.Empty(), - AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? Array.Empty(), - HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? Array.Empty(), - UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), - DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), - DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? Array.Empty(), - DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption), + Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? Array.Empty(), + HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? Array.Empty(), + RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), + ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), - WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), - Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", + Affinity = parseResult.GetValue(CommandLineOptions.AffinityOption), DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), - StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", - - CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? Array.Empty(), - CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), - RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null - ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) - : null, - ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), - ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", - - RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), + AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? Array.Empty(), + AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? Array.Empty(), + AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? Array.Empty(), Join = parseResult.GetValue(CommandLineOptions.JoinOption), KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), - StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), - DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), - LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), - GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), - ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), - Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), - TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), - + HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? Array.Empty(), + CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), + RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null + ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) + : null, + CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? Array.Empty(), + MonoPath = parseResult.GetValue(CommandLineOptions.MonoPathOption), + ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption), + ILCompilerVersion = parseResult.GetValue(CommandLineOptions.ILCompilerVersionOption), + IlcPackages = parseResult.GetValue(CommandLineOptions.IlcPackagesOption), LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), WarmupIterationCount = parseResult.GetValue(CommandLineOptions.WarmupCountOption), + MinWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MinWarmupCountOption), + MaxWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MaxWarmupCountOption), + IterationTimeInMilliseconds = parseResult.GetValue(CommandLineOptions.IterationTimeOption), IterationCount = parseResult.GetValue(CommandLineOptions.IterationCountOption), + MinIterationCount = parseResult.GetValue(CommandLineOptions.MinIterationCountOption), + MaxIterationCount = parseResult.GetValue(CommandLineOptions.MaxIterationCountOption), InvocationCount = parseResult.GetValue(CommandLineOptions.InvocationCountOption), UnrollFactor = parseResult.GetValue(CommandLineOptions.UnrollFactorOption), + RunStrategy = parseResult.GetValue(CommandLineOptions.RunStrategyOption), + Platform = parseResult.GetValue(CommandLineOptions.PlatformOption), RunOncePerIteration = parseResult.GetValue(CommandLineOptions.RunOnceOption), + PrintInformation = parseResult.GetValue(CommandLineOptions.PrintInformationOption), + ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), + ListBenchmarkCaseMode = parseResult.GetValue(CommandLineOptions.ListBenchmarkCaseModeOption), + DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), + DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? Array.Empty(), + DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), + GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), + TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), + WakeLock = parseResult.GetValue(CommandLineOptions.WakeLockOption), + StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), + StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption), + DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), + MaxParameterColumnWidth = parseResult.GetValue(CommandLineOptions.MaxParameterColumnWidthOption), + EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? Array.Empty(), MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), - NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption) + WasmJavascriptEngine = parseResult.GetValue(CommandLineOptions.WasmJavascriptEngineOption), + WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), + CustomRuntimePack = parseResult.GetValue(CommandLineOptions.CustomRuntimePackOption), + AOTCompilerPath = parseResult.GetValue(CommandLineOptions.AOTCompilerPathOption), + AOTCompilerMode = parseResult.GetValue(CommandLineOptions.AOTCompilerModeOption), + WasmDataDirectory = parseResult.GetValue(CommandLineOptions.WasmDataDirectoryOption), + WasmCoreCLR = parseResult.GetValue(CommandLineOptions.WasmCoreCLROption), + NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), + NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), + Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), }; bool isSuccess = Validate(options, logger); From cba362d1c72202273e5cafa19c75c9b517b37841 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Fri, 20 Feb 2026 13:29:59 +0200 Subject: [PATCH 07/18] style: apply review nits, fix help ordering, and support --version --- .../ConsoleArguments/CommandLineOptions.cs | 6 +- .../ConsoleArguments/ConfigParser.cs | 172 +++++++++--------- 2 files changed, 90 insertions(+), 88 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 84f1e72f27..a068034a7c 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -241,7 +241,7 @@ public bool UseDisassemblyDiagnoser get => useDisassemblyDiagnoser || DisassemblerRecursiveDepth != DefaultDisassemblerRecursiveDepth || DisassemblerFilters.Any(); set => useDisassemblyDiagnoser = value; } - public string? Profiler { get; set; } + public string Profiler { get; set; } = ""; public IEnumerable Filters { get; set; } = []; public IEnumerable HiddenColumns { get; set; } = []; public bool RunInProcess { get; set; } @@ -260,7 +260,7 @@ public bool UseDisassemblyDiagnoser public DirectoryInfo? RestorePath { get; set; } public IReadOnlyList CoreRunPaths { get; set; } = []; public FileInfo? MonoPath { get; set; } - public string? ClrVersion { get; set; } + public string ClrVersion { get; set; } = ""; public string? ILCompilerVersion { get; set; } public DirectoryInfo? IlcPackages { get; set; } public int? LaunchCount { get; set; } @@ -287,7 +287,7 @@ public bool UseDisassemblyDiagnoser public int? TimeOutInSeconds { get; set; } public WakeLockType? WakeLock { get; set; } public bool StopOnFirstError { get; set; } - public string? StatisticalTestThreshold { get; set; } + public string StatisticalTestThreshold { get; set; } = ""; public bool DisableLogFile { get; set; } public int? MaxParameterColumnWidth { get; set; } public IEnumerable EnvironmentVariables { get; set; } = []; diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 4a7c42d89c..c64d3fd730 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -81,81 +81,83 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par if (!expandSuccess) return (false, default, default); args = expandedArgs; - var rootCommand = new RootCommand("BenchmarkDotNet Command Line options"); - rootCommand.TreatUnmatchedTokensAsErrors = false; + var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") + { + CommandLineOptions.BaseJobOption, + CommandLineOptions.RuntimesOption, + CommandLineOptions.ExportersOption, + CommandLineOptions.MemoryOption, + CommandLineOptions.ThreadingOption, + CommandLineOptions.ExceptionsOption, + CommandLineOptions.DisassemblyOption, + CommandLineOptions.ProfilerOption, + CommandLineOptions.FiltersOption, + CommandLineOptions.HiddenColumnsOption, + CommandLineOptions.RunInProcessOption, + CommandLineOptions.ArtifactsDirectoryOption, + CommandLineOptions.OutliersOption, + CommandLineOptions.AffinityOption, + CommandLineOptions.DisplayAllStatisticsOption, + CommandLineOptions.AllCategoriesOption, + CommandLineOptions.AnyCategoriesOption, + CommandLineOptions.AttributeNamesOption, + CommandLineOptions.JoinOption, + CommandLineOptions.KeepBenchmarkFilesOption, + CommandLineOptions.DontOverwriteResultsOption, + CommandLineOptions.HardwareCountersOption, + CommandLineOptions.CliPathOption, + CommandLineOptions.RestorePathOption, + CommandLineOptions.CoreRunPathsOption, + CommandLineOptions.MonoPathOption, + CommandLineOptions.ClrVersionOption, + CommandLineOptions.ILCompilerVersionOption, + CommandLineOptions.IlcPackagesOption, + CommandLineOptions.LaunchCountOption, + CommandLineOptions.WarmupCountOption, + CommandLineOptions.MinWarmupCountOption, + CommandLineOptions.MaxWarmupCountOption, + CommandLineOptions.IterationTimeOption, + CommandLineOptions.IterationCountOption, + CommandLineOptions.MinIterationCountOption, + CommandLineOptions.MaxIterationCountOption, + CommandLineOptions.InvocationCountOption, + CommandLineOptions.UnrollFactorOption, + CommandLineOptions.RunStrategyOption, + CommandLineOptions.PlatformOption, + CommandLineOptions.RunOnceOption, + CommandLineOptions.PrintInformationOption, + CommandLineOptions.ApplesToApplesOption, + CommandLineOptions.ListBenchmarkCaseModeOption, + CommandLineOptions.DisassemblerDepthOption, + CommandLineOptions.DisassemblerFiltersOption, + CommandLineOptions.DisassemblerDiffOption, + CommandLineOptions.LogBuildOutputOption, + CommandLineOptions.GenerateBinLogOption, + CommandLineOptions.TimeoutOption, + CommandLineOptions.WakeLockOption, + CommandLineOptions.StopOnFirstErrorOption, + CommandLineOptions.StatisticalTestThresholdOption, + CommandLineOptions.DisableLogFileOption, + CommandLineOptions.MaxParameterColumnWidthOption, + CommandLineOptions.EnvironmentVariablesOption, + CommandLineOptions.MemoryRandomizationOption, + CommandLineOptions.WasmJavascriptEngineOption, + CommandLineOptions.WasmJavaScriptEngineArgumentsOption, + CommandLineOptions.CustomRuntimePackOption, + CommandLineOptions.AOTCompilerPathOption, + CommandLineOptions.AOTCompilerModeOption, + CommandLineOptions.WasmDataDirectoryOption, + CommandLineOptions.WasmCoreCLROption, + CommandLineOptions.NoForcedGCsOption, + CommandLineOptions.NoEvaluationOverheadOption, + CommandLineOptions.ResumeOption, + }; - rootCommand.Add(CommandLineOptions.BaseJobOption); - rootCommand.Add(CommandLineOptions.RuntimesOption); - rootCommand.Add(CommandLineOptions.ExportersOption); - rootCommand.Add(CommandLineOptions.MemoryOption); - rootCommand.Add(CommandLineOptions.ThreadingOption); - rootCommand.Add(CommandLineOptions.ExceptionsOption); - rootCommand.Add(CommandLineOptions.DisassemblyOption); - rootCommand.Add(CommandLineOptions.ProfilerOption); - rootCommand.Add(CommandLineOptions.FiltersOption); - rootCommand.Add(CommandLineOptions.HiddenColumnsOption); - rootCommand.Add(CommandLineOptions.RunInProcessOption); - rootCommand.Add(CommandLineOptions.ArtifactsDirectoryOption); - rootCommand.Add(CommandLineOptions.OutliersOption); - rootCommand.Add(CommandLineOptions.AffinityOption); - rootCommand.Add(CommandLineOptions.DisplayAllStatisticsOption); - rootCommand.Add(CommandLineOptions.AllCategoriesOption); - rootCommand.Add(CommandLineOptions.AnyCategoriesOption); - rootCommand.Add(CommandLineOptions.AttributeNamesOption); - rootCommand.Add(CommandLineOptions.JoinOption); - rootCommand.Add(CommandLineOptions.KeepBenchmarkFilesOption); - rootCommand.Add(CommandLineOptions.DontOverwriteResultsOption); - rootCommand.Add(CommandLineOptions.HardwareCountersOption); - rootCommand.Add(CommandLineOptions.CliPathOption); - rootCommand.Add(CommandLineOptions.RestorePathOption); - rootCommand.Add(CommandLineOptions.CoreRunPathsOption); - rootCommand.Add(CommandLineOptions.MonoPathOption); - rootCommand.Add(CommandLineOptions.ClrVersionOption); - rootCommand.Add(CommandLineOptions.ILCompilerVersionOption); - rootCommand.Add(CommandLineOptions.IlcPackagesOption); - rootCommand.Add(CommandLineOptions.LaunchCountOption); - rootCommand.Add(CommandLineOptions.WarmupCountOption); - rootCommand.Add(CommandLineOptions.MinWarmupCountOption); - rootCommand.Add(CommandLineOptions.MaxWarmupCountOption); - rootCommand.Add(CommandLineOptions.IterationTimeOption); - rootCommand.Add(CommandLineOptions.IterationCountOption); - rootCommand.Add(CommandLineOptions.MinIterationCountOption); - rootCommand.Add(CommandLineOptions.MaxIterationCountOption); - rootCommand.Add(CommandLineOptions.InvocationCountOption); - rootCommand.Add(CommandLineOptions.UnrollFactorOption); - rootCommand.Add(CommandLineOptions.RunStrategyOption); - rootCommand.Add(CommandLineOptions.PlatformOption); - rootCommand.Add(CommandLineOptions.RunOnceOption); - rootCommand.Add(CommandLineOptions.PrintInformationOption); - rootCommand.Add(CommandLineOptions.ApplesToApplesOption); - rootCommand.Add(CommandLineOptions.ListBenchmarkCaseModeOption); - rootCommand.Add(CommandLineOptions.DisassemblerDepthOption); - rootCommand.Add(CommandLineOptions.DisassemblerFiltersOption); - rootCommand.Add(CommandLineOptions.DisassemblerDiffOption); - rootCommand.Add(CommandLineOptions.LogBuildOutputOption); - rootCommand.Add(CommandLineOptions.GenerateBinLogOption); - rootCommand.Add(CommandLineOptions.TimeoutOption); - rootCommand.Add(CommandLineOptions.WakeLockOption); - rootCommand.Add(CommandLineOptions.StopOnFirstErrorOption); - rootCommand.Add(CommandLineOptions.StatisticalTestThresholdOption); - rootCommand.Add(CommandLineOptions.DisableLogFileOption); - rootCommand.Add(CommandLineOptions.MaxParameterColumnWidthOption); - rootCommand.Add(CommandLineOptions.EnvironmentVariablesOption); - rootCommand.Add(CommandLineOptions.MemoryRandomizationOption); - rootCommand.Add(CommandLineOptions.WasmJavascriptEngineOption); - rootCommand.Add(CommandLineOptions.WasmJavaScriptEngineArgumentsOption); - rootCommand.Add(CommandLineOptions.CustomRuntimePackOption); - rootCommand.Add(CommandLineOptions.AOTCompilerPathOption); - rootCommand.Add(CommandLineOptions.AOTCompilerModeOption); - rootCommand.Add(CommandLineOptions.WasmDataDirectoryOption); - rootCommand.Add(CommandLineOptions.WasmCoreCLROption); - rootCommand.Add(CommandLineOptions.NoForcedGCsOption); - rootCommand.Add(CommandLineOptions.NoEvaluationOverheadOption); - rootCommand.Add(CommandLineOptions.ResumeOption); + rootCommand.TreatUnmatchedTokensAsErrors = false; var parseResult = rootCommand.Parse(args); - if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) + if (args.Any(a => a == "-h" || a == "--help" || a == "-?" || a == "--version")) { parseResult.Invoke(); return (false, default, default); @@ -164,34 +166,34 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par var options = new CommandLineOptions { BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", - Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? Array.Empty(), - Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? Array.Empty(), + Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], + Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), - Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption), - Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? Array.Empty(), - HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? Array.Empty(), + Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", + Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? [], + HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), Affinity = parseResult.GetValue(CommandLineOptions.AffinityOption), DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), - AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? Array.Empty(), - AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? Array.Empty(), - AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? Array.Empty(), + AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? [], + AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? [], + AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? [], Join = parseResult.GetValue(CommandLineOptions.JoinOption), KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), - HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? Array.Empty(), + HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? [], CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) : null, - CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? Array.Empty(), + CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? [], MonoPath = parseResult.GetValue(CommandLineOptions.MonoPathOption), - ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption), + ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", ILCompilerVersion = parseResult.GetValue(CommandLineOptions.ILCompilerVersionOption), IlcPackages = parseResult.GetValue(CommandLineOptions.IlcPackagesOption), LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), @@ -211,17 +213,17 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), ListBenchmarkCaseMode = parseResult.GetValue(CommandLineOptions.ListBenchmarkCaseModeOption), DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), - DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? Array.Empty(), + DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? [], DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), WakeLock = parseResult.GetValue(CommandLineOptions.WakeLockOption), StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), - StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption), + StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), MaxParameterColumnWidth = parseResult.GetValue(CommandLineOptions.MaxParameterColumnWidthOption), - EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? Array.Empty(), + EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? [], MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), WasmJavascriptEngine = parseResult.GetValue(CommandLineOptions.WasmJavascriptEngineOption), WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), From 110ac8fd2dd9d1087d0c228531abc87b7bd2a713 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Fri, 20 Feb 2026 20:58:33 +0200 Subject: [PATCH 08/18] feat: implement argument normalization and serialization for command line options --- .../ConsoleArguments/ConfigParser.cs | 464 +++++++++++++++++- 1 file changed, 459 insertions(+), 5 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index c64d3fd730..6481cdf163 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -33,6 +33,7 @@ using System.CommandLine.Parsing; using System.Runtime.InteropServices; using System.CommandLine.Invocation; +using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; using RuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; namespace BenchmarkDotNet.ConsoleArguments @@ -75,12 +76,137 @@ public static class ConfigParser { "fullxml", new[] { XmlExporter.Full } } }; + private static string[] NormalizeArgs(string[] args) + { + var aliasToCanonical = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + // short aliases + ["-j"] = "--job", + ["-r"] = "--runtimes", + ["-e"] = "--exporters", + ["-m"] = "--memory", + ["-t"] = "--threading", + ["-d"] = "--disasm", + ["-p"] = "--profiler", + ["-f"] = "--filter", + ["-h"] = "--hide", + ["-i"] = "--inProcess", + ["-a"] = "--artifacts", + + ["--job"] = "--job", + ["--runtimes"] = "--runtimes", + ["--exporters"] = "--exporters", + ["--memory"] = "--memory", + ["--threading"] = "--threading", + ["--exceptions"] = "--exceptions", + ["--disasm"] = "--disasm", + ["--profiler"] = "--profiler", + ["--filter"] = "--filter", + ["--hide"] = "--hide", + ["--inprocess"] = "--inProcess", + ["--artifacts"] = "--artifacts", + ["--outliers"] = "--outliers", + ["--affinity"] = "--affinity", + ["--allstats"] = "--allStats", + ["--allcategories"] = "--allCategories", + ["--anycategories"] = "--anyCategories", + ["--attribute"] = "--attribute", + ["--join"] = "--join", + ["--keepfiles"] = "--keepFiles", + ["--nooverwrite"] = "--noOverwrite", + ["--counters"] = "--counters", + ["--cli"] = "--cli", + ["--packages"] = "--packages", + ["--corerun"] = "--coreRun", + ["--monopath"] = "--monoPath", + ["--clrversion"] = "--clrVersion", + ["--ilcompilerversion"] = "--ilCompilerVersion", + ["--ilcpackages"] = "--ilcPackages", + ["--launchcount"] = "--launchCount", + ["--warmupcount"] = "--warmupCount", + ["--minwarmupcount"] = "--minWarmupCount", + ["--maxwarmupcount"] = "--maxWarmupCount", + ["--iterationtime"] = "--iterationTime", + ["--iterationcount"] = "--iterationCount", + ["--miniterationcount"] = "--minIterationCount", + ["--maxiterationcount"] = "--maxIterationCount", + ["--invocationcount"] = "--invocationCount", + ["--unrollfactor"] = "--unrollFactor", + ["--strategy"] = "--strategy", + ["--platform"] = "--platform", + ["--runOncePerIteration"] = "--runOncePerIteration", + ["--runoncperiteration"] = "--runOncePerIteration", + ["--info"] = "--info", + ["--apples"] = "--apples", + ["--list"] = "--list", + ["--disasmdepth"] = "--disasmDepth", + ["--disasmfilter"] = "--disasmFilter", + ["--disasmdiff"] = "--disasmDiff", + ["--logbuildoutput"] = "--logBuildOutput", + ["--generatebinlog"] = "--generateBinLog", + ["--buildtimeout"] = "--buildTimeout", + ["--wakelock"] = "--wakeLock", + ["--stoponfirsterror"] = "--stopOnFirstError", + ["--statisticaltest"] = "--statisticalTest", + ["--disablelogfile"] = "--disableLogFile", + ["--maxwidth"] = "--maxWidth", + ["--envvars"] = "--envVars", + ["--memoryrandomization"] = "--memoryRandomization", + ["--wasmengine"] = "--wasmEngine", + ["--wasmargs"] = "--wasmArgs", + ["--customruntimepack"] = "--customRuntimePack", + ["--aotcompilerpath"] = "--AOTCompilerPath", + ["--aotcompilermode"] = "--AOTCompilerMode", + ["--wasmdatadir"] = "--wasmDataDir", + ["--wasmcoreclr"] = "--wasmCoreCLR", + ["--noforcedgcs"] = "--noForcedGCs", + ["--nooverheadevaluation"] = "--noOverheadEvaluation", + ["--resume"] = "--resume", + }; + + var result = new List(); + for (int i = 0; i < args.Length; i++) + { + var arg = args[i]; + + if (arg == "--") + { + result.Add(arg); + for (int j = i + 1; j < args.Length; j++) + result.Add(args[j]); + break; + } + + if (arg.StartsWith("-")) + { + var eqIdx = arg.IndexOf('='); + string key = eqIdx >= 0 ? arg.Substring(0, eqIdx) : arg; + string? value = eqIdx >= 0 ? arg.Substring(eqIdx + 1) : null; + + if (aliasToCanonical.TryGetValue(key, out var canonical)) + key = canonical; + + if (key.Equals("--counters", StringComparison.OrdinalIgnoreCase) && value != null && value.Contains('+')) + { + result.Add(key); + result.AddRange(value.Split('+')); + continue; + } + + arg = value != null ? $"{key}={value}" : key; + } + + result.Add(arg); + } + + return result.ToArray(); + } public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) { var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); if (!expandSuccess) return (false, default, default); args = expandedArgs; - + args = NormalizeArgs(args); var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") { CommandLineOptions.BaseJobOption, @@ -345,13 +471,156 @@ string GetToken() } internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) { - var rootCommand = new RootCommand("BenchmarkDotNet Command Line options"); - rootCommand.Add(CommandLineOptions.BaseJobOption); + args = NormalizeArgs(args); + + var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") + { + CommandLineOptions.BaseJobOption, + CommandLineOptions.RuntimesOption, + CommandLineOptions.ExportersOption, + CommandLineOptions.MemoryOption, + CommandLineOptions.ThreadingOption, + CommandLineOptions.ExceptionsOption, + CommandLineOptions.DisassemblyOption, + CommandLineOptions.ProfilerOption, + CommandLineOptions.FiltersOption, + CommandLineOptions.HiddenColumnsOption, + CommandLineOptions.RunInProcessOption, + CommandLineOptions.ArtifactsDirectoryOption, + CommandLineOptions.OutliersOption, + CommandLineOptions.AffinityOption, + CommandLineOptions.DisplayAllStatisticsOption, + CommandLineOptions.AllCategoriesOption, + CommandLineOptions.AnyCategoriesOption, + CommandLineOptions.AttributeNamesOption, + CommandLineOptions.JoinOption, + CommandLineOptions.KeepBenchmarkFilesOption, + CommandLineOptions.DontOverwriteResultsOption, + CommandLineOptions.HardwareCountersOption, + CommandLineOptions.CliPathOption, + CommandLineOptions.RestorePathOption, + CommandLineOptions.CoreRunPathsOption, + CommandLineOptions.MonoPathOption, + CommandLineOptions.ClrVersionOption, + CommandLineOptions.ILCompilerVersionOption, + CommandLineOptions.IlcPackagesOption, + CommandLineOptions.LaunchCountOption, + CommandLineOptions.WarmupCountOption, + CommandLineOptions.MinWarmupCountOption, + CommandLineOptions.MaxWarmupCountOption, + CommandLineOptions.IterationTimeOption, + CommandLineOptions.IterationCountOption, + CommandLineOptions.MinIterationCountOption, + CommandLineOptions.MaxIterationCountOption, + CommandLineOptions.InvocationCountOption, + CommandLineOptions.UnrollFactorOption, + CommandLineOptions.RunStrategyOption, + CommandLineOptions.PlatformOption, + CommandLineOptions.RunOnceOption, + CommandLineOptions.PrintInformationOption, + CommandLineOptions.ApplesToApplesOption, + CommandLineOptions.ListBenchmarkCaseModeOption, + CommandLineOptions.DisassemblerDepthOption, + CommandLineOptions.DisassemblerFiltersOption, + CommandLineOptions.DisassemblerDiffOption, + CommandLineOptions.LogBuildOutputOption, + CommandLineOptions.GenerateBinLogOption, + CommandLineOptions.TimeoutOption, + CommandLineOptions.WakeLockOption, + CommandLineOptions.StopOnFirstErrorOption, + CommandLineOptions.StatisticalTestThresholdOption, + CommandLineOptions.DisableLogFileOption, + CommandLineOptions.MaxParameterColumnWidthOption, + CommandLineOptions.EnvironmentVariablesOption, + CommandLineOptions.MemoryRandomizationOption, + CommandLineOptions.WasmJavascriptEngineOption, + CommandLineOptions.WasmJavaScriptEngineArgumentsOption, + CommandLineOptions.CustomRuntimePackOption, + CommandLineOptions.AOTCompilerPathOption, + CommandLineOptions.AOTCompilerModeOption, + CommandLineOptions.WasmDataDirectoryOption, + CommandLineOptions.WasmCoreCLROption, + CommandLineOptions.NoForcedGCsOption, + CommandLineOptions.NoEvaluationOverheadOption, + CommandLineOptions.ResumeOption, + }; + + rootCommand.TreatUnmatchedTokensAsErrors = false; var parseResult = rootCommand.Parse(args); + var options = new CommandLineOptions { - BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "" + BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", + Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], + Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], + UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), + UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), + UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), + UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), + Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", + Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? [], + HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], + RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), + ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), + Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), + Affinity = parseResult.GetValue(CommandLineOptions.AffinityOption), + DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), + AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? [], + AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? [], + AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? [], + Join = parseResult.GetValue(CommandLineOptions.JoinOption), + KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), + DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), + HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? [], + CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), + RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null + ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) + : null, + CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? [], + MonoPath = parseResult.GetValue(CommandLineOptions.MonoPathOption), + ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", + ILCompilerVersion = parseResult.GetValue(CommandLineOptions.ILCompilerVersionOption), + IlcPackages = parseResult.GetValue(CommandLineOptions.IlcPackagesOption), + LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), + WarmupIterationCount = parseResult.GetValue(CommandLineOptions.WarmupCountOption), + MinWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MinWarmupCountOption), + MaxWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MaxWarmupCountOption), + IterationTimeInMilliseconds = parseResult.GetValue(CommandLineOptions.IterationTimeOption), + IterationCount = parseResult.GetValue(CommandLineOptions.IterationCountOption), + MinIterationCount = parseResult.GetValue(CommandLineOptions.MinIterationCountOption), + MaxIterationCount = parseResult.GetValue(CommandLineOptions.MaxIterationCountOption), + InvocationCount = parseResult.GetValue(CommandLineOptions.InvocationCountOption), + UnrollFactor = parseResult.GetValue(CommandLineOptions.UnrollFactorOption), + RunStrategy = parseResult.GetValue(CommandLineOptions.RunStrategyOption), + Platform = parseResult.GetValue(CommandLineOptions.PlatformOption), + RunOncePerIteration = parseResult.GetValue(CommandLineOptions.RunOnceOption), + PrintInformation = parseResult.GetValue(CommandLineOptions.PrintInformationOption), + ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), + ListBenchmarkCaseMode = parseResult.GetValue(CommandLineOptions.ListBenchmarkCaseModeOption), + DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), + DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? [], + DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), + GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), + TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), + WakeLock = parseResult.GetValue(CommandLineOptions.WakeLockOption), + StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), + StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", + DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), + MaxParameterColumnWidth = parseResult.GetValue(CommandLineOptions.MaxParameterColumnWidthOption), + EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? [], + MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), + WasmJavascriptEngine = parseResult.GetValue(CommandLineOptions.WasmJavascriptEngineOption), + WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), + CustomRuntimePack = parseResult.GetValue(CommandLineOptions.CustomRuntimePackOption), + AOTCompilerPath = parseResult.GetValue(CommandLineOptions.AOTCompilerPathOption), + AOTCompilerMode = parseResult.GetValue(CommandLineOptions.AOTCompilerModeOption), + WasmDataDirectory = parseResult.GetValue(CommandLineOptions.WasmDataDirectoryOption), + WasmCoreCLR = parseResult.GetValue(CommandLineOptions.WasmCoreCLROption), + NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), + NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), + Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), }; if (!Validate(options, NullLogger.Instance)) @@ -361,10 +630,195 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act } updater(options); - updatedArgs = args; + + updatedArgs = SerializeToArgs(options); return true; } + private static string[] SerializeToArgs(CommandLineOptions options) + { + var result = new List(); + + // --filter (no default) + if (options.Filters.Any()) + { + result.Add("--filter"); + result.AddRange(options.Filters); + } + + if (options.BaseJob.IsNotBlank() && !options.BaseJob.Equals("Default", StringComparison.OrdinalIgnoreCase)) + { + result.Add("--job"); + result.Add(options.BaseJob); + } + + // --runtimes + if (options.Runtimes.Any()) + { + result.Add("--runtimes"); + result.AddRange(options.Runtimes); + } + + if (options.UseMemoryDiagnoser) result.Add("--memory"); + if (options.UseThreadingDiagnoser) result.Add("--threading"); + if (options.UseExceptionDiagnoser) result.Add("--exceptions"); + if (options.UseDisassemblyDiagnoser) result.Add("--disasm"); + if (options.RunInProcess) result.Add("--inProcess"); + if (options.Join) result.Add("--join"); + if (options.KeepBenchmarkFiles) result.Add("--keepFiles"); + if (options.DontOverwriteResults) result.Add("--noOverwrite"); + if (options.DisplayAllStatistics) result.Add("--allStats"); + if (options.RunOncePerIteration) result.Add("--runOncePerIteration"); + if (options.PrintInformation) result.Add("--info"); + if (options.ApplesToApples) result.Add("--apples"); + if (options.LogBuildOutput) result.Add("--logBuildOutput"); + if (options.GenerateMSBuildBinLog) result.Add("--generateBinLog"); + if (options.StopOnFirstError) result.Add("--stopOnFirstError"); + if (options.DisableLogFile) result.Add("--disableLogFile"); + if (options.MemoryRandomization) result.Add("--memoryRandomization"); + if (options.DisassemblerDiff) result.Add("--disasmDiff"); + if (options.WasmCoreCLR) result.Add("--wasmCoreCLR"); + if (options.NoForcedGCs) result.Add("--noForcedGCs"); + if (options.NoEvaluationOverhead) result.Add("--noOverheadEvaluation"); + if (options.Resume) result.Add("--resume"); + + // strings + if (options.Profiler.IsNotBlank()) + { result.Add("--profiler"); result.Add(options.Profiler); } + + if (options.ClrVersion.IsNotBlank()) + { result.Add("--clrVersion"); result.Add(options.ClrVersion); } + + if (options.StatisticalTestThreshold.IsNotBlank()) + { result.Add("--statisticalTest"); result.Add(options.StatisticalTestThreshold); } + + if (options.ILCompilerVersion.IsNotBlank()) + { result.Add("--ilCompilerVersion"); result.Add(options.ILCompilerVersion!); } + + if (options.CustomRuntimePack.IsNotBlank()) + { result.Add("--customRuntimePack"); result.Add(options.CustomRuntimePack!); } + + if (options.WasmJavaScriptEngineArguments != null + && options.WasmJavaScriptEngineArguments != "--expose_wasm") + { result.Add("--wasmArgs"); result.Add(options.WasmJavaScriptEngineArguments); } + + // collections + if (options.Exporters.Any()) + { result.Add("--exporters"); result.AddRange(options.Exporters); } + + if (options.HardwareCounters.Any()) + { result.Add("--counters"); result.AddRange(options.HardwareCounters); } + + if (options.AllCategories.Any()) + { result.Add("--allCategories"); result.AddRange(options.AllCategories); } + + if (options.AnyCategories.Any()) + { result.Add("--anyCategories"); result.AddRange(options.AnyCategories); } + + if (options.AttributeNames.Any()) + { result.Add("--attribute"); result.AddRange(options.AttributeNames); } + + if (options.HiddenColumns.Any()) + { result.Add("--hide"); result.AddRange(options.HiddenColumns); } + + if (options.DisassemblerFilters.Any()) + { result.Add("--disasmFilter"); result.AddRange(options.DisassemblerFilters); } + + if (options.EnvironmentVariables.Any()) + { result.Add("--envVars"); result.AddRange(options.EnvironmentVariables); } + + // nullable ints/long + if (options.LaunchCount.HasValue) + { result.Add("--launchCount"); result.Add(options.LaunchCount.Value.ToString()); } + + if (options.WarmupIterationCount.HasValue) + { result.Add("--warmupCount"); result.Add(options.WarmupIterationCount.Value.ToString()); } + + if (options.MinWarmupIterationCount.HasValue) + { result.Add("--minWarmupCount"); result.Add(options.MinWarmupIterationCount.Value.ToString()); } + + if (options.MaxWarmupIterationCount.HasValue) + { result.Add("--maxWarmupCount"); result.Add(options.MaxWarmupIterationCount.Value.ToString()); } + + if (options.IterationTimeInMilliseconds.HasValue) + { result.Add("--iterationTime"); result.Add(options.IterationTimeInMilliseconds.Value.ToString()); } + + if (options.IterationCount.HasValue) + { result.Add("--iterationCount"); result.Add(options.IterationCount.Value.ToString()); } + + if (options.MinIterationCount.HasValue) + { result.Add("--minIterationCount"); result.Add(options.MinIterationCount.Value.ToString()); } + + if (options.MaxIterationCount.HasValue) + { result.Add("--maxIterationCount"); result.Add(options.MaxIterationCount.Value.ToString()); } + + if (options.InvocationCount.HasValue) + { result.Add("--invocationCount"); result.Add(options.InvocationCount.Value.ToString()); } + + if (options.UnrollFactor.HasValue) + { result.Add("--unrollFactor"); result.Add(options.UnrollFactor.Value.ToString()); } + + if (options.Affinity.HasValue) + { result.Add("--affinity"); result.Add(options.Affinity.Value.ToString()); } + + if (options.TimeOutInSeconds.HasValue) + { result.Add("--buildTimeout"); result.Add(options.TimeOutInSeconds.Value.ToString()); } + + if (options.MaxParameterColumnWidth.HasValue) + { result.Add("--maxWidth"); result.Add(options.MaxParameterColumnWidth.Value.ToString()); } + + if (options.DisassemblerRecursiveDepth != 1) + { result.Add("--disasmDepth"); result.Add(options.DisassemblerRecursiveDepth.ToString()); } + + if (options.Outliers != OutlierMode.RemoveUpper) + { result.Add("--outliers"); result.Add(options.Outliers.ToString()); } + + if (options.RunStrategy.HasValue) + { result.Add("--strategy"); result.Add(options.RunStrategy.Value.ToString()); } + + if (options.Platform.HasValue) + { result.Add("--platform"); result.Add(options.Platform.Value.ToString()); } + + if (options.WakeLock.HasValue) + { result.Add("--wakeLock"); result.Add(options.WakeLock.Value.ToString()); } + + if (options.ListBenchmarkCaseMode != ListBenchmarkCaseMode.Disabled) + { result.Add("--list"); result.Add(options.ListBenchmarkCaseMode.ToString()); } + + if (options.AOTCompilerMode != MonoAotCompilerMode.mini) + { result.Add("--AOTCompilerMode"); result.Add(options.AOTCompilerMode.ToString()); } + + // files/directories + if (options.ArtifactsDirectory != null) + { result.Add("--artifacts"); result.Add(options.ArtifactsDirectory.FullName); } + + if (options.CliPath != null) + { result.Add("--cli"); result.Add(options.CliPath.FullName); } + + if (options.RestorePath != null) + { result.Add("--packages"); result.Add(options.RestorePath.FullName); } + + if (options.MonoPath != null) + { result.Add("--monoPath"); result.Add(options.MonoPath.FullName); } + + if (options.IlcPackages != null) + { result.Add("--ilcPackages"); result.Add(options.IlcPackages.FullName); } + + if (options.WasmJavascriptEngine != null) + { result.Add("--wasmEngine"); result.Add(options.WasmJavascriptEngine.FullName); } + + if (options.WasmDataDirectory != null) + { result.Add("--wasmDataDir"); result.Add(options.WasmDataDirectory.FullName); } + + if (options.AOTCompilerPath != null) + { result.Add("--AOTCompilerPath"); result.Add(options.AOTCompilerPath.FullName); } + + if (options.CoreRunPaths.Any()) + { result.Add("--coreRun"); result.AddRange(options.CoreRunPaths.Select(p => p.FullName)); } + + return result.ToArray(); + } + private static bool Validate(CommandLineOptions options, ILogger logger) { if (options.BaseJob.IsBlank() || !AvailableJobs.ContainsKey(options.BaseJob)) From cf09e3e1e32ff0143fb80f9b60cda56baab57311 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Fri, 20 Feb 2026 21:28:01 +0200 Subject: [PATCH 09/18] fix: enable multiple args per token, handle unmatched tokens as errors, and fix serialization order --- .../ConsoleArguments/CommandLineOptions.cs | 17 +++++++++++ .../ConsoleArguments/ConfigParser.cs | 29 +++++++++++++------ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index a068034a7c..6096aade87 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -305,5 +305,22 @@ public bool UseDisassemblyDiagnoser internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any(); private static string Escape(string input) => UserInteractionHelper.EscapeCommandExample(input); + + static CommandLineOptions() + { + // Allow space-separated arrays (e.g. --exporters html rplot) + RuntimesOption.AllowMultipleArgumentsPerToken = true; + ExportersOption.AllowMultipleArgumentsPerToken = true; + FiltersOption.AllowMultipleArgumentsPerToken = true; + AllCategoriesOption.AllowMultipleArgumentsPerToken = true; + AnyCategoriesOption.AllowMultipleArgumentsPerToken = true; + AttributeNamesOption.AllowMultipleArgumentsPerToken = true; + HiddenColumnsOption.AllowMultipleArgumentsPerToken = true; + HardwareCountersOption.AllowMultipleArgumentsPerToken = true; + EnvironmentVariablesOption.AllowMultipleArgumentsPerToken = true; + DisassemblerFiltersOption.AllowMultipleArgumentsPerToken = true; + CoreRunPathsOption.AllowMultipleArgumentsPerToken = true; + } + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 6481cdf163..52b65ab78d 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -289,6 +289,10 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par return (false, default, default); } + var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-") && t != "--").ToList(); + if (invalidOptions.Any()) + return (false, default, default); + var options = new CommandLineOptions { BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", @@ -299,7 +303,9 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", - Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? [], + Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) + .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-") && t != "--")) + .ToArray(), HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), @@ -549,6 +555,8 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act var parseResult = rootCommand.Parse(args); + var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-") && t != "--").ToList(); + var options = new CommandLineOptions { BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", @@ -559,7 +567,9 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", - Filters = parseResult.GetValue(CommandLineOptions.FiltersOption) ?? [], + Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) + .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-") && t != "--")) + .ToArray(), HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), @@ -623,7 +633,7 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), }; - if (!Validate(options, NullLogger.Instance)) + if (invalidOptions.Any() || !Validate(options, NullLogger.Instance)) { updatedArgs = null; return false; @@ -652,12 +662,6 @@ private static string[] SerializeToArgs(CommandLineOptions options) result.Add(options.BaseJob); } - // --runtimes - if (options.Runtimes.Any()) - { - result.Add("--runtimes"); - result.AddRange(options.Runtimes); - } if (options.UseMemoryDiagnoser) result.Add("--memory"); if (options.UseThreadingDiagnoser) result.Add("--threading"); @@ -682,6 +686,13 @@ private static string[] SerializeToArgs(CommandLineOptions options) if (options.NoEvaluationOverhead) result.Add("--noOverheadEvaluation"); if (options.Resume) result.Add("--resume"); + // --runtimes + if (options.Runtimes.Any()) + { + result.Add("--runtimes"); + result.AddRange(options.Runtimes); + } + // strings if (options.Profiler.IsNotBlank()) { result.Add("--profiler"); result.Add(options.Profiler); } From 9d0d233f1413279ec9d08a3dacb211f9b8efe667 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sat, 21 Feb 2026 06:34:33 +0200 Subject: [PATCH 10/18] feat: add support for extra command line arguments and improve error handling for unrecognized options --- .../ConsoleArguments/CommandLineOptions.cs | 22 ++++++++ .../ConsoleArguments/ConfigParser.cs | 56 +++++++++++++------ 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 6096aade87..a275971e82 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -302,6 +302,7 @@ public bool UseDisassemblyDiagnoser public bool NoForcedGCs { get; set; } public bool NoEvaluationOverhead { get; set; } public bool Resume { get; set; } + public string[] ExtraArguments { get; set; } = []; internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any(); private static string Escape(string input) => UserInteractionHelper.EscapeCommandExample(input); @@ -320,6 +321,27 @@ static CommandLineOptions() EnvironmentVariablesOption.AllowMultipleArgumentsPerToken = true; DisassemblerFiltersOption.AllowMultipleArgumentsPerToken = true; CoreRunPathsOption.AllowMultipleArgumentsPerToken = true; + + void AddUnrecognizedValidator(Option option) + { + option.Validators.Add(result => + { + foreach (var token in result.Tokens.Where(t => t.Value.StartsWith("-", StringComparison.Ordinal))) + result.AddError($"Unrecognized option: {token.Value}"); + }); + } + + AddUnrecognizedValidator(RuntimesOption); + AddUnrecognizedValidator(ExportersOption); + AddUnrecognizedValidator(FiltersOption); + AddUnrecognizedValidator(AllCategoriesOption); + AddUnrecognizedValidator(AnyCategoriesOption); + AddUnrecognizedValidator(AttributeNamesOption); + AddUnrecognizedValidator(HiddenColumnsOption); + AddUnrecognizedValidator(HardwareCountersOption); + AddUnrecognizedValidator(EnvironmentVariablesOption); + AddUnrecognizedValidator(DisassemblerFiltersOption); + AddUnrecognizedValidator(CoreRunPathsOption); } } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 52b65ab78d..3a68a24922 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -186,10 +186,18 @@ private static string[] NormalizeArgs(string[] args) if (aliasToCanonical.TryGetValue(key, out var canonical)) key = canonical; - if (key.Equals("--counters", StringComparison.OrdinalIgnoreCase) && value != null && value.Contains('+')) + if (key.Equals("--counters", StringComparison.OrdinalIgnoreCase)) { result.Add(key); - result.AddRange(value.Split('+')); + if (value != null) + { + result.AddRange(value.Split('+')); + } + else if (i + 1 < args.Length && !args[i + 1].StartsWith("-") && args[i + 1].Contains('+')) + { + i++; + result.AddRange(args[i].Split('+')); + } continue; } @@ -201,12 +209,22 @@ private static string[] NormalizeArgs(string[] args) return result.ToArray(); } + public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) { var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); if (!expandSuccess) return (false, default, default); args = expandedArgs; args = NormalizeArgs(args); + + string[] extraArgs = []; + var dashDashIndex = Array.IndexOf(args, "--"); + if (dashDashIndex >= 0) + { + extraArgs = args.Skip(dashDashIndex + 1).ToArray(); + args = args.Take(dashDashIndex).ToArray(); + } + var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") { CommandLineOptions.BaseJobOption, @@ -279,8 +297,6 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par CommandLineOptions.ResumeOption, }; - rootCommand.TreatUnmatchedTokensAsErrors = false; - var parseResult = rootCommand.Parse(args); if (args.Any(a => a == "-h" || a == "--help" || a == "-?" || a == "--version")) @@ -289,12 +305,22 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par return (false, default, default); } + if (parseResult.Errors.Any()) + { + foreach (var error in parseResult.Errors) + { + logger.WriteLineError(error.Message); + } + return (false, default, default); + } + var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-") && t != "--").ToList(); if (invalidOptions.Any()) return (false, default, default); var options = new CommandLineOptions { + ExtraArguments = extraArgs, BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], @@ -479,6 +505,14 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act { args = NormalizeArgs(args); + string[] extraArgs = []; + var dashDashIndex = Array.IndexOf(args, "--"); + if (dashDashIndex >= 0) + { + extraArgs = args.Skip(dashDashIndex + 1).ToArray(); + args = args.Take(dashDashIndex).ToArray(); + } + var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") { CommandLineOptions.BaseJobOption, @@ -551,7 +585,6 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act CommandLineOptions.ResumeOption, }; - rootCommand.TreatUnmatchedTokensAsErrors = false; var parseResult = rootCommand.Parse(args); @@ -559,6 +592,7 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act var options = new CommandLineOptions { + ExtraArguments = extraArgs, BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], @@ -1329,18 +1363,6 @@ private static IEnumerable GetFilters(CommandLineOptions options) yield return new AttributesFilter(options.AttributeNames.ToArray()); } - private static int GetMaximumDisplayWidth() - { - try - { - return Console.WindowWidth; - } - catch (IOException) - { - return MinimumDisplayWidth; - } - } - private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, FileInfo coreRunPath) => baseJob .WithToolchain(new CoreRunToolchain( From 26e4348ab4f623608c6259d43d327d9c99aa3518 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sat, 21 Feb 2026 15:46:29 +0200 Subject: [PATCH 11/18] feat: add duplicate option detection and error handling for command line arguments --- .../ConsoleArguments/ConfigParser.cs | 277 +++++++++--------- 1 file changed, 141 insertions(+), 136 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 3a68a24922..bf008daab4 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -66,7 +66,7 @@ public static class ConfigParser { "stackoverflow", new[] { MarkdownExporter.StackOverflow } }, { "github", new[] { MarkdownExporter.GitHub } }, { "plain", new[] { PlainExporter.Default } }, - { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, // R Plots depends on having the full measurements available + { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, { "json", new[] { JsonExporter.Default } }, { "briefjson", new[] { JsonExporter.Brief } }, { "fulljson", new[] { JsonExporter.Full } }, @@ -76,11 +76,34 @@ public static class ConfigParser { "fullxml", new[] { XmlExporter.Full } } }; + private static bool HasDuplicateOptions(string[] args) + { + var aliasToCanonical = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["-j"] = "--job", + ["-r"] = "--runtimes", + ["-e"] = "--exporters", + ["-m"] = "--memory", + ["-t"] = "--threading", + ["-d"] = "--disasm", + ["-p"] = "--profiler", + ["-f"] = "--filter", + ["-h"] = "--hide", + ["-i"] = "--inprocess", + ["-a"] = "--artifacts" + }; + + var options = args.Where(a => a.StartsWith("-") && a != "--") + .Select(a => a.Split('=')[0].ToLowerInvariant()) + .Select(a => aliasToCanonical.TryGetValue(a, out var c) ? c : a); + + return options.GroupBy(x => x).Any(g => g.Count() > 1); + } + private static string[] NormalizeArgs(string[] args) { var aliasToCanonical = new Dictionary(StringComparer.OrdinalIgnoreCase) { - // short aliases ["-j"] = "--job", ["-r"] = "--runtimes", ["-e"] = "--exporters", @@ -212,6 +235,14 @@ private static string[] NormalizeArgs(string[] args) public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) { + args = args.Where(a => !string.IsNullOrWhiteSpace(a)).ToArray(); + + if (HasDuplicateOptions(args)) + { + logger.WriteLineError("Duplicate options are not allowed."); + return (false, default, default); + } + var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); if (!expandSuccess) return (false, default, default); args = expandedArgs; @@ -316,7 +347,11 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-") && t != "--").ToList(); if (invalidOptions.Any()) + { + foreach (var opt in invalidOptions) + logger.WriteLineError($"Option '{opt.TrimStart('-')}' is unknown."); return (false, default, default); + } var options = new CommandLineOptions { @@ -501,8 +536,17 @@ string GetToken() return result; } } + internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) { + args = args.Where(a => !string.IsNullOrWhiteSpace(a)).ToArray(); + + if (HasDuplicateOptions(args)) + { + updatedArgs = null; + return false; + } + args = NormalizeArgs(args); string[] extraArgs = []; @@ -514,77 +558,76 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act } var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") - { - CommandLineOptions.BaseJobOption, - CommandLineOptions.RuntimesOption, - CommandLineOptions.ExportersOption, - CommandLineOptions.MemoryOption, - CommandLineOptions.ThreadingOption, - CommandLineOptions.ExceptionsOption, - CommandLineOptions.DisassemblyOption, - CommandLineOptions.ProfilerOption, - CommandLineOptions.FiltersOption, - CommandLineOptions.HiddenColumnsOption, - CommandLineOptions.RunInProcessOption, - CommandLineOptions.ArtifactsDirectoryOption, - CommandLineOptions.OutliersOption, - CommandLineOptions.AffinityOption, - CommandLineOptions.DisplayAllStatisticsOption, - CommandLineOptions.AllCategoriesOption, - CommandLineOptions.AnyCategoriesOption, - CommandLineOptions.AttributeNamesOption, - CommandLineOptions.JoinOption, - CommandLineOptions.KeepBenchmarkFilesOption, - CommandLineOptions.DontOverwriteResultsOption, - CommandLineOptions.HardwareCountersOption, - CommandLineOptions.CliPathOption, - CommandLineOptions.RestorePathOption, - CommandLineOptions.CoreRunPathsOption, - CommandLineOptions.MonoPathOption, - CommandLineOptions.ClrVersionOption, - CommandLineOptions.ILCompilerVersionOption, - CommandLineOptions.IlcPackagesOption, - CommandLineOptions.LaunchCountOption, - CommandLineOptions.WarmupCountOption, - CommandLineOptions.MinWarmupCountOption, - CommandLineOptions.MaxWarmupCountOption, - CommandLineOptions.IterationTimeOption, - CommandLineOptions.IterationCountOption, - CommandLineOptions.MinIterationCountOption, - CommandLineOptions.MaxIterationCountOption, - CommandLineOptions.InvocationCountOption, - CommandLineOptions.UnrollFactorOption, - CommandLineOptions.RunStrategyOption, - CommandLineOptions.PlatformOption, - CommandLineOptions.RunOnceOption, - CommandLineOptions.PrintInformationOption, - CommandLineOptions.ApplesToApplesOption, - CommandLineOptions.ListBenchmarkCaseModeOption, - CommandLineOptions.DisassemblerDepthOption, - CommandLineOptions.DisassemblerFiltersOption, - CommandLineOptions.DisassemblerDiffOption, - CommandLineOptions.LogBuildOutputOption, - CommandLineOptions.GenerateBinLogOption, - CommandLineOptions.TimeoutOption, - CommandLineOptions.WakeLockOption, - CommandLineOptions.StopOnFirstErrorOption, - CommandLineOptions.StatisticalTestThresholdOption, - CommandLineOptions.DisableLogFileOption, - CommandLineOptions.MaxParameterColumnWidthOption, - CommandLineOptions.EnvironmentVariablesOption, - CommandLineOptions.MemoryRandomizationOption, - CommandLineOptions.WasmJavascriptEngineOption, - CommandLineOptions.WasmJavaScriptEngineArgumentsOption, - CommandLineOptions.CustomRuntimePackOption, - CommandLineOptions.AOTCompilerPathOption, - CommandLineOptions.AOTCompilerModeOption, - CommandLineOptions.WasmDataDirectoryOption, - CommandLineOptions.WasmCoreCLROption, - CommandLineOptions.NoForcedGCsOption, - CommandLineOptions.NoEvaluationOverheadOption, - CommandLineOptions.ResumeOption, - }; - + { + CommandLineOptions.BaseJobOption, + CommandLineOptions.RuntimesOption, + CommandLineOptions.ExportersOption, + CommandLineOptions.MemoryOption, + CommandLineOptions.ThreadingOption, + CommandLineOptions.ExceptionsOption, + CommandLineOptions.DisassemblyOption, + CommandLineOptions.ProfilerOption, + CommandLineOptions.FiltersOption, + CommandLineOptions.HiddenColumnsOption, + CommandLineOptions.RunInProcessOption, + CommandLineOptions.ArtifactsDirectoryOption, + CommandLineOptions.OutliersOption, + CommandLineOptions.AffinityOption, + CommandLineOptions.DisplayAllStatisticsOption, + CommandLineOptions.AllCategoriesOption, + CommandLineOptions.AnyCategoriesOption, + CommandLineOptions.AttributeNamesOption, + CommandLineOptions.JoinOption, + CommandLineOptions.KeepBenchmarkFilesOption, + CommandLineOptions.DontOverwriteResultsOption, + CommandLineOptions.HardwareCountersOption, + CommandLineOptions.CliPathOption, + CommandLineOptions.RestorePathOption, + CommandLineOptions.CoreRunPathsOption, + CommandLineOptions.MonoPathOption, + CommandLineOptions.ClrVersionOption, + CommandLineOptions.ILCompilerVersionOption, + CommandLineOptions.IlcPackagesOption, + CommandLineOptions.LaunchCountOption, + CommandLineOptions.WarmupCountOption, + CommandLineOptions.MinWarmupCountOption, + CommandLineOptions.MaxWarmupCountOption, + CommandLineOptions.IterationTimeOption, + CommandLineOptions.IterationCountOption, + CommandLineOptions.MinIterationCountOption, + CommandLineOptions.MaxIterationCountOption, + CommandLineOptions.InvocationCountOption, + CommandLineOptions.UnrollFactorOption, + CommandLineOptions.RunStrategyOption, + CommandLineOptions.PlatformOption, + CommandLineOptions.RunOnceOption, + CommandLineOptions.PrintInformationOption, + CommandLineOptions.ApplesToApplesOption, + CommandLineOptions.ListBenchmarkCaseModeOption, + CommandLineOptions.DisassemblerDepthOption, + CommandLineOptions.DisassemblerFiltersOption, + CommandLineOptions.DisassemblerDiffOption, + CommandLineOptions.LogBuildOutputOption, + CommandLineOptions.GenerateBinLogOption, + CommandLineOptions.TimeoutOption, + CommandLineOptions.WakeLockOption, + CommandLineOptions.StopOnFirstErrorOption, + CommandLineOptions.StatisticalTestThresholdOption, + CommandLineOptions.DisableLogFileOption, + CommandLineOptions.MaxParameterColumnWidthOption, + CommandLineOptions.EnvironmentVariablesOption, + CommandLineOptions.MemoryRandomizationOption, + CommandLineOptions.WasmJavascriptEngineOption, + CommandLineOptions.WasmJavaScriptEngineArgumentsOption, + CommandLineOptions.CustomRuntimePackOption, + CommandLineOptions.AOTCompilerPathOption, + CommandLineOptions.AOTCompilerModeOption, + CommandLineOptions.WasmDataDirectoryOption, + CommandLineOptions.WasmCoreCLROption, + CommandLineOptions.NoForcedGCsOption, + CommandLineOptions.NoEvaluationOverheadOption, + CommandLineOptions.ResumeOption, + }; var parseResult = rootCommand.Parse(args); @@ -683,7 +726,6 @@ private static string[] SerializeToArgs(CommandLineOptions options) { var result = new List(); - // --filter (no default) if (options.Filters.Any()) { result.Add("--filter"); @@ -696,7 +738,6 @@ private static string[] SerializeToArgs(CommandLineOptions options) result.Add(options.BaseJob); } - if (options.UseMemoryDiagnoser) result.Add("--memory"); if (options.UseThreadingDiagnoser) result.Add("--threading"); if (options.UseExceptionDiagnoser) result.Add("--exceptions"); @@ -720,14 +761,12 @@ private static string[] SerializeToArgs(CommandLineOptions options) if (options.NoEvaluationOverhead) result.Add("--noOverheadEvaluation"); if (options.Resume) result.Add("--resume"); - // --runtimes if (options.Runtimes.Any()) { result.Add("--runtimes"); result.AddRange(options.Runtimes); } - // strings if (options.Profiler.IsNotBlank()) { result.Add("--profiler"); result.Add(options.Profiler); } @@ -747,7 +786,6 @@ private static string[] SerializeToArgs(CommandLineOptions options) && options.WasmJavaScriptEngineArguments != "--expose_wasm") { result.Add("--wasmArgs"); result.Add(options.WasmJavaScriptEngineArguments); } - // collections if (options.Exporters.Any()) { result.Add("--exporters"); result.AddRange(options.Exporters); } @@ -772,7 +810,6 @@ private static string[] SerializeToArgs(CommandLineOptions options) if (options.EnvironmentVariables.Any()) { result.Add("--envVars"); result.AddRange(options.EnvironmentVariables); } - // nullable ints/long if (options.LaunchCount.HasValue) { result.Add("--launchCount"); result.Add(options.LaunchCount.Value.ToString()); } @@ -833,7 +870,6 @@ private static string[] SerializeToArgs(CommandLineOptions options) if (options.AOTCompilerMode != MonoAotCompilerMode.mini) { result.Add("--AOTCompilerMode"); result.Add(options.AOTCompilerMode.ToString()); } - // files/directories if (options.ArtifactsDirectory != null) { result.Add("--artifacts"); result.Add(options.ArtifactsDirectory.FullName); } @@ -964,9 +1000,9 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC var config = new ManualConfig(); var baseJob = GetBaseJob(options, globalConfig); - var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID + var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); if (expanded.Length > 1) - expanded[0] = expanded[0].AsBaseline(); // if the user provides multiple jobs, then the first one should be a baseline + expanded[0] = expanded[0].AsBaseline(); config.AddJob(expanded); if (config.GetJobs().IsEmpty() && baseJob != Job.Default) config.AddJob(baseJob); @@ -1035,7 +1071,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC private static Job GetBaseJob(CommandLineOptions options, IConfig? globalConfig) { var baseJob = - globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) // global config might define single custom Default job + globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) ?? AvailableJobs[options.BaseJob.ToLowerInvariant()]; if (baseJob != Job.Dry && options.Outliers != OutlierMode.RemoveUpper) @@ -1075,9 +1111,9 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig? globalConfig) if (options.NoForcedGCs) baseJob = baseJob.WithGcForce(false); if (options.NoEvaluationOverhead) -#pragma warning disable CS0618 // Type or member is obsolete +#pragma warning disable CS0618 baseJob = baseJob.WithEvaluateOverhead(false); -#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning restore CS0618 if (options.EnvironmentVariables.Any()) { @@ -1088,12 +1124,12 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig? globalConfig) }).ToArray()); } - if (AvailableJobs.Values.Contains(baseJob)) // no custom settings + if (AvailableJobs.Values.Contains(baseJob)) return baseJob; return baseJob - .AsDefault(false) // after applying all settings from console args the base job is not default anymore - .AsMutator(); // we mark it as mutator so it will be applied to other jobs defined via attributes and merged later in GetRunnableJobs method + .AsDefault(false) + .AsMutator(); } private static IEnumerable Expand(Job baseJob, CommandLineOptions options, string[] args) @@ -1105,7 +1141,7 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, else if (options.ClrVersion.IsNotBlank()) { var runtime = ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion); - yield return baseJob.WithRuntime(runtime).WithId(runtime.Name); // local builds of .NET Runtime + yield return baseJob.WithRuntime(runtime).WithId(runtime.Name); } else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty()) { @@ -1113,28 +1149,26 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, } else { - // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job string? first = args.FirstOrDefault(arg => arg.Equals("--runtimes", StringComparison.OrdinalIgnoreCase) || arg.Equals("-r", StringComparison.OrdinalIgnoreCase) - || arg.Equals("--corerun", StringComparison.OrdinalIgnoreCase)); if (first is null || first.Equals("--corerun", StringComparison.OrdinalIgnoreCase)) { foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + yield return CreateCoreRunJob(baseJob, options, coreRunPath); - foreach (string runtime in options.Runtimes) // known runtimes + foreach (string runtime in options.Runtimes) yield return CreateJobForGivenRuntime(baseJob, runtime, options); } else { - foreach (string runtime in options.Runtimes) // known runtimes + foreach (string runtime in options.Runtimes) yield return CreateJobForGivenRuntime(baseJob, runtime, options); foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + yield return CreateCoreRunJob(baseJob, options, coreRunPath); } } } @@ -1196,61 +1230,45 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.NativeAot60: return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); - case RuntimeMoniker.NativeAot70: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); - case RuntimeMoniker.NativeAot80: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); - case RuntimeMoniker.NativeAot90: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"); - case RuntimeMoniker.NativeAot10_0: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"); - case RuntimeMoniker.NativeAot11_0: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet11/nuget/v3/index.json"); case RuntimeMoniker.WasmNet80: return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); - case RuntimeMoniker.WasmNet90: return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); - case RuntimeMoniker.WasmNet10_0: return MakeWasmJob(baseJob, options, "net10.0", runtimeMoniker); - case RuntimeMoniker.WasmNet11_0: return MakeWasmJob(baseJob, options, "net11.0", runtimeMoniker); case RuntimeMoniker.MonoAOTLLVM: return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet60: return MakeMonoAOTLLVMJob(baseJob, options, "net6.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet70: return MakeMonoAOTLLVMJob(baseJob, options, "net7.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet80: return MakeMonoAOTLLVMJob(baseJob, options, "net8.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet90: return MakeMonoAOTLLVMJob(baseJob, options, "net9.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet10_0: return MakeMonoAOTLLVMJob(baseJob, options, "net10.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet11_0: return MakeMonoAOTLLVMJob(baseJob, options, "net11.0", runtimeMoniker); case RuntimeMoniker.Mono60: return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); - case RuntimeMoniker.Mono70: return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); - case RuntimeMoniker.Mono80: return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); @@ -1308,11 +1326,11 @@ private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, s moniker: moniker); var toolChain = MonoAotLLVMToolChain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: monoAotLLVMRuntime.Name, - options: options)); + new NetCoreAppSettings( + targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, + runtimeFrameworkVersion: "", + name: monoAotLLVMRuntime.Name, + options: options)); return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain).WithId(monoAotLLVMRuntime.Name); } @@ -1320,11 +1338,11 @@ private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, s private static Job CreateR2RJob(Job baseJob, CommandLineOptions options, Runtime runtime) { var toolChain = R2RToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: runtime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: runtime.Name, - options: options)); + new NetCoreAppSettings( + targetFrameworkMoniker: runtime.MsBuildMoniker, + runtimeFrameworkVersion: "", + name: runtime.Name, + options: options)); return baseJob.WithRuntime(runtime).WithToolchain(toolChain).WithId(runtime.Name); } @@ -1371,7 +1389,7 @@ private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, Fil targetFrameworkMoniker: RuntimeInformation.IsNetCore ? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker - : CoreRuntime.Latest.MsBuildMoniker, // use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors + : CoreRuntime.Latest.MsBuildMoniker, customDotNetCliPath: options.CliPath, restorePath: options.RestorePath, displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath))); @@ -1385,18 +1403,6 @@ private static Job CreateCoreJobWithCli(Job baseJob, CommandLineOptions options) name: RuntimeInformation.GetCurrentRuntime().Name, options: options))); - /// - /// we have a limited amount of space when printing the output to the console, so we try to keep things small and simple - /// - /// for following paths: - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe - /// - /// we get: - /// - /// \Core_Root\CoreRun.exe - /// \Core_Root_beforeMyChanges\CoreRun.exe - /// private static string GetCoreRunToolchainDisplayName(IReadOnlyList paths, FileInfo coreRunPath) { if (paths.Count <= 1) @@ -1430,7 +1436,6 @@ internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) runtime = runtime.Substring(0, index); } - // Monikers older than Net 10 don't use any version delimiter, newer monikers use _ delimiter. if (Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker)) { return true; @@ -1438,4 +1443,4 @@ internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) return Enum.TryParse(runtime.Replace('.', '_'), ignoreCase: true, out runtimeMoniker); } } -} +} \ No newline at end of file From 5e23e876daec7541ab00a2092782aef6c988fc3e Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sat, 21 Feb 2026 17:09:55 +0200 Subject: [PATCH 12/18] feat: enhance error reporting for unknown command line options --- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index bf008daab4..4d220ea8bb 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -340,11 +340,19 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par { foreach (var error in parseResult.Errors) { - logger.WriteLineError(error.Message); + string msg = error.Message; + + var badArg = args.FirstOrDefault(a => a.StartsWith("-") && msg.Contains(a)); + + if (badArg != null) + { + msg = $"Option '{badArg.TrimStart('-')}' is unknown."; + } + + logger.WriteLineError(msg); } return (false, default, default); } - var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-") && t != "--").ToList(); if (invalidOptions.Any()) { From 8ff8652988294529ef65f1a9931ff97ae4a8accb Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sun, 22 Feb 2026 13:35:54 +0200 Subject: [PATCH 13/18] feat: add implicit filters argument and refactor command line options handling --- .../ConsoleArguments/ConfigParser.cs | 323 ++++++------------ .../Helpers/CultureInfoHelper.cs | 33 ++ 2 files changed, 135 insertions(+), 221 deletions(-) create mode 100644 src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 4d220ea8bb..2fdbbd8c41 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -76,6 +76,94 @@ public static class ConfigParser { "fullxml", new[] { XmlExporter.Full } } }; + internal static Argument ImplicitFiltersArgument { get; } = new Argument("implicitFilters") { Arity = ArgumentArity.ZeroOrMore }; + + private static RootCommand? _rootCommand; + internal static RootCommand RootCommand + { + get + { + if (_rootCommand == null) + { + using var invariantUICultureScope = Helpers.CultureInfoHelper.CreateInvariantUICultureScope(); + + _rootCommand = new RootCommand("BenchmarkDotNet Command Line options") + { + CommandLineOptions.BaseJobOption, + CommandLineOptions.RuntimesOption, + CommandLineOptions.ExportersOption, + CommandLineOptions.MemoryOption, + CommandLineOptions.ThreadingOption, + CommandLineOptions.ExceptionsOption, + CommandLineOptions.DisassemblyOption, + CommandLineOptions.ProfilerOption, + CommandLineOptions.FiltersOption, + CommandLineOptions.HiddenColumnsOption, + CommandLineOptions.RunInProcessOption, + CommandLineOptions.ArtifactsDirectoryOption, + CommandLineOptions.OutliersOption, + CommandLineOptions.AffinityOption, + CommandLineOptions.DisplayAllStatisticsOption, + CommandLineOptions.AllCategoriesOption, + CommandLineOptions.AnyCategoriesOption, + CommandLineOptions.AttributeNamesOption, + CommandLineOptions.JoinOption, + CommandLineOptions.KeepBenchmarkFilesOption, + CommandLineOptions.DontOverwriteResultsOption, + CommandLineOptions.HardwareCountersOption, + CommandLineOptions.CliPathOption, + CommandLineOptions.RestorePathOption, + CommandLineOptions.CoreRunPathsOption, + CommandLineOptions.MonoPathOption, + CommandLineOptions.ClrVersionOption, + CommandLineOptions.ILCompilerVersionOption, + CommandLineOptions.IlcPackagesOption, + CommandLineOptions.LaunchCountOption, + CommandLineOptions.WarmupCountOption, + CommandLineOptions.MinWarmupCountOption, + CommandLineOptions.MaxWarmupCountOption, + CommandLineOptions.IterationTimeOption, + CommandLineOptions.IterationCountOption, + CommandLineOptions.MinIterationCountOption, + CommandLineOptions.MaxIterationCountOption, + CommandLineOptions.InvocationCountOption, + CommandLineOptions.UnrollFactorOption, + CommandLineOptions.RunStrategyOption, + CommandLineOptions.PlatformOption, + CommandLineOptions.RunOnceOption, + CommandLineOptions.PrintInformationOption, + CommandLineOptions.ApplesToApplesOption, + CommandLineOptions.ListBenchmarkCaseModeOption, + CommandLineOptions.DisassemblerDepthOption, + CommandLineOptions.DisassemblerFiltersOption, + CommandLineOptions.DisassemblerDiffOption, + CommandLineOptions.LogBuildOutputOption, + CommandLineOptions.GenerateBinLogOption, + CommandLineOptions.TimeoutOption, + CommandLineOptions.WakeLockOption, + CommandLineOptions.StopOnFirstErrorOption, + CommandLineOptions.StatisticalTestThresholdOption, + CommandLineOptions.DisableLogFileOption, + CommandLineOptions.MaxParameterColumnWidthOption, + CommandLineOptions.EnvironmentVariablesOption, + CommandLineOptions.MemoryRandomizationOption, + CommandLineOptions.WasmJavascriptEngineOption, + CommandLineOptions.WasmJavaScriptEngineArgumentsOption, + CommandLineOptions.CustomRuntimePackOption, + CommandLineOptions.AOTCompilerPathOption, + CommandLineOptions.AOTCompilerModeOption, + CommandLineOptions.WasmDataDirectoryOption, + CommandLineOptions.WasmCoreCLROption, + CommandLineOptions.NoForcedGCsOption, + CommandLineOptions.NoEvaluationOverheadOption, + CommandLineOptions.ResumeOption, + }; + _rootCommand.Add(ImplicitFiltersArgument); + } + return _rootCommand; + } + } + private static bool HasDuplicateOptions(string[] args) { var aliasToCanonical = new Dictionary(StringComparer.OrdinalIgnoreCase) @@ -256,82 +344,11 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par args = args.Take(dashDashIndex).ToArray(); } - var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") - { - CommandLineOptions.BaseJobOption, - CommandLineOptions.RuntimesOption, - CommandLineOptions.ExportersOption, - CommandLineOptions.MemoryOption, - CommandLineOptions.ThreadingOption, - CommandLineOptions.ExceptionsOption, - CommandLineOptions.DisassemblyOption, - CommandLineOptions.ProfilerOption, - CommandLineOptions.FiltersOption, - CommandLineOptions.HiddenColumnsOption, - CommandLineOptions.RunInProcessOption, - CommandLineOptions.ArtifactsDirectoryOption, - CommandLineOptions.OutliersOption, - CommandLineOptions.AffinityOption, - CommandLineOptions.DisplayAllStatisticsOption, - CommandLineOptions.AllCategoriesOption, - CommandLineOptions.AnyCategoriesOption, - CommandLineOptions.AttributeNamesOption, - CommandLineOptions.JoinOption, - CommandLineOptions.KeepBenchmarkFilesOption, - CommandLineOptions.DontOverwriteResultsOption, - CommandLineOptions.HardwareCountersOption, - CommandLineOptions.CliPathOption, - CommandLineOptions.RestorePathOption, - CommandLineOptions.CoreRunPathsOption, - CommandLineOptions.MonoPathOption, - CommandLineOptions.ClrVersionOption, - CommandLineOptions.ILCompilerVersionOption, - CommandLineOptions.IlcPackagesOption, - CommandLineOptions.LaunchCountOption, - CommandLineOptions.WarmupCountOption, - CommandLineOptions.MinWarmupCountOption, - CommandLineOptions.MaxWarmupCountOption, - CommandLineOptions.IterationTimeOption, - CommandLineOptions.IterationCountOption, - CommandLineOptions.MinIterationCountOption, - CommandLineOptions.MaxIterationCountOption, - CommandLineOptions.InvocationCountOption, - CommandLineOptions.UnrollFactorOption, - CommandLineOptions.RunStrategyOption, - CommandLineOptions.PlatformOption, - CommandLineOptions.RunOnceOption, - CommandLineOptions.PrintInformationOption, - CommandLineOptions.ApplesToApplesOption, - CommandLineOptions.ListBenchmarkCaseModeOption, - CommandLineOptions.DisassemblerDepthOption, - CommandLineOptions.DisassemblerFiltersOption, - CommandLineOptions.DisassemblerDiffOption, - CommandLineOptions.LogBuildOutputOption, - CommandLineOptions.GenerateBinLogOption, - CommandLineOptions.TimeoutOption, - CommandLineOptions.WakeLockOption, - CommandLineOptions.StopOnFirstErrorOption, - CommandLineOptions.StatisticalTestThresholdOption, - CommandLineOptions.DisableLogFileOption, - CommandLineOptions.MaxParameterColumnWidthOption, - CommandLineOptions.EnvironmentVariablesOption, - CommandLineOptions.MemoryRandomizationOption, - CommandLineOptions.WasmJavascriptEngineOption, - CommandLineOptions.WasmJavaScriptEngineArgumentsOption, - CommandLineOptions.CustomRuntimePackOption, - CommandLineOptions.AOTCompilerPathOption, - CommandLineOptions.AOTCompilerModeOption, - CommandLineOptions.WasmDataDirectoryOption, - CommandLineOptions.WasmCoreCLROption, - CommandLineOptions.NoForcedGCsOption, - CommandLineOptions.NoEvaluationOverheadOption, - CommandLineOptions.ResumeOption, - }; - - var parseResult = rootCommand.Parse(args); + var parseResult = RootCommand.Parse(args); if (args.Any(a => a == "-h" || a == "--help" || a == "-?" || a == "--version")) { + using var invariantUICultureScope = Helpers.CultureInfoHelper.CreateInvariantUICultureScope(); parseResult.Invoke(); return (false, default, default); } @@ -341,19 +358,17 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par foreach (var error in parseResult.Errors) { string msg = error.Message; - var badArg = args.FirstOrDefault(a => a.StartsWith("-") && msg.Contains(a)); - if (badArg != null) { msg = $"Option '{badArg.TrimStart('-')}' is unknown."; } - logger.WriteLineError(msg); } return (false, default, default); } - var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-") && t != "--").ToList(); + + var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-")).ToList(); if (invalidOptions.Any()) { foreach (var opt in invalidOptions) @@ -373,7 +388,7 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) - .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-") && t != "--")) + .Concat(parseResult.GetValue(ImplicitFiltersArgument) ?? []) .ToArray(), HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), @@ -565,160 +580,26 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act args = args.Take(dashDashIndex).ToArray(); } - var rootCommand = new RootCommand("BenchmarkDotNet Command Line options") - { - CommandLineOptions.BaseJobOption, - CommandLineOptions.RuntimesOption, - CommandLineOptions.ExportersOption, - CommandLineOptions.MemoryOption, - CommandLineOptions.ThreadingOption, - CommandLineOptions.ExceptionsOption, - CommandLineOptions.DisassemblyOption, - CommandLineOptions.ProfilerOption, - CommandLineOptions.FiltersOption, - CommandLineOptions.HiddenColumnsOption, - CommandLineOptions.RunInProcessOption, - CommandLineOptions.ArtifactsDirectoryOption, - CommandLineOptions.OutliersOption, - CommandLineOptions.AffinityOption, - CommandLineOptions.DisplayAllStatisticsOption, - CommandLineOptions.AllCategoriesOption, - CommandLineOptions.AnyCategoriesOption, - CommandLineOptions.AttributeNamesOption, - CommandLineOptions.JoinOption, - CommandLineOptions.KeepBenchmarkFilesOption, - CommandLineOptions.DontOverwriteResultsOption, - CommandLineOptions.HardwareCountersOption, - CommandLineOptions.CliPathOption, - CommandLineOptions.RestorePathOption, - CommandLineOptions.CoreRunPathsOption, - CommandLineOptions.MonoPathOption, - CommandLineOptions.ClrVersionOption, - CommandLineOptions.ILCompilerVersionOption, - CommandLineOptions.IlcPackagesOption, - CommandLineOptions.LaunchCountOption, - CommandLineOptions.WarmupCountOption, - CommandLineOptions.MinWarmupCountOption, - CommandLineOptions.MaxWarmupCountOption, - CommandLineOptions.IterationTimeOption, - CommandLineOptions.IterationCountOption, - CommandLineOptions.MinIterationCountOption, - CommandLineOptions.MaxIterationCountOption, - CommandLineOptions.InvocationCountOption, - CommandLineOptions.UnrollFactorOption, - CommandLineOptions.RunStrategyOption, - CommandLineOptions.PlatformOption, - CommandLineOptions.RunOnceOption, - CommandLineOptions.PrintInformationOption, - CommandLineOptions.ApplesToApplesOption, - CommandLineOptions.ListBenchmarkCaseModeOption, - CommandLineOptions.DisassemblerDepthOption, - CommandLineOptions.DisassemblerFiltersOption, - CommandLineOptions.DisassemblerDiffOption, - CommandLineOptions.LogBuildOutputOption, - CommandLineOptions.GenerateBinLogOption, - CommandLineOptions.TimeoutOption, - CommandLineOptions.WakeLockOption, - CommandLineOptions.StopOnFirstErrorOption, - CommandLineOptions.StatisticalTestThresholdOption, - CommandLineOptions.DisableLogFileOption, - CommandLineOptions.MaxParameterColumnWidthOption, - CommandLineOptions.EnvironmentVariablesOption, - CommandLineOptions.MemoryRandomizationOption, - CommandLineOptions.WasmJavascriptEngineOption, - CommandLineOptions.WasmJavaScriptEngineArgumentsOption, - CommandLineOptions.CustomRuntimePackOption, - CommandLineOptions.AOTCompilerPathOption, - CommandLineOptions.AOTCompilerModeOption, - CommandLineOptions.WasmDataDirectoryOption, - CommandLineOptions.WasmCoreCLROption, - CommandLineOptions.NoForcedGCsOption, - CommandLineOptions.NoEvaluationOverheadOption, - CommandLineOptions.ResumeOption, - }; + var parseResult = RootCommand.Parse(args); - var parseResult = rootCommand.Parse(args); + var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-")).ToList(); - var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-") && t != "--").ToList(); + if (parseResult.Errors.Any() || invalidOptions.Any()) + { + updatedArgs = null; + return false; + } var options = new CommandLineOptions { ExtraArguments = extraArgs, BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", - Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], - Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], - UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), - UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), - UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), - UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), - Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) - .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-") && t != "--")) + .Concat(parseResult.GetValue(ImplicitFiltersArgument) ?? []) .ToArray(), - HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], - RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), - ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), - Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), - Affinity = parseResult.GetValue(CommandLineOptions.AffinityOption), - DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), - AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? [], - AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? [], - AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? [], - Join = parseResult.GetValue(CommandLineOptions.JoinOption), - KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), - DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), - HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? [], - CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), - RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null - ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) - : null, - CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? [], - MonoPath = parseResult.GetValue(CommandLineOptions.MonoPathOption), - ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", - ILCompilerVersion = parseResult.GetValue(CommandLineOptions.ILCompilerVersionOption), - IlcPackages = parseResult.GetValue(CommandLineOptions.IlcPackagesOption), - LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), - WarmupIterationCount = parseResult.GetValue(CommandLineOptions.WarmupCountOption), - MinWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MinWarmupCountOption), - MaxWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MaxWarmupCountOption), - IterationTimeInMilliseconds = parseResult.GetValue(CommandLineOptions.IterationTimeOption), - IterationCount = parseResult.GetValue(CommandLineOptions.IterationCountOption), - MinIterationCount = parseResult.GetValue(CommandLineOptions.MinIterationCountOption), - MaxIterationCount = parseResult.GetValue(CommandLineOptions.MaxIterationCountOption), - InvocationCount = parseResult.GetValue(CommandLineOptions.InvocationCountOption), - UnrollFactor = parseResult.GetValue(CommandLineOptions.UnrollFactorOption), - RunStrategy = parseResult.GetValue(CommandLineOptions.RunStrategyOption), - Platform = parseResult.GetValue(CommandLineOptions.PlatformOption), - RunOncePerIteration = parseResult.GetValue(CommandLineOptions.RunOnceOption), - PrintInformation = parseResult.GetValue(CommandLineOptions.PrintInformationOption), - ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), - ListBenchmarkCaseMode = parseResult.GetValue(CommandLineOptions.ListBenchmarkCaseModeOption), - DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), - DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? [], - DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), - LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), - GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), - TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), - WakeLock = parseResult.GetValue(CommandLineOptions.WakeLockOption), - StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), - StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", - DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), - MaxParameterColumnWidth = parseResult.GetValue(CommandLineOptions.MaxParameterColumnWidthOption), - EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? [], - MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), - WasmJavascriptEngine = parseResult.GetValue(CommandLineOptions.WasmJavascriptEngineOption), - WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), - CustomRuntimePack = parseResult.GetValue(CommandLineOptions.CustomRuntimePackOption), - AOTCompilerPath = parseResult.GetValue(CommandLineOptions.AOTCompilerPathOption), - AOTCompilerMode = parseResult.GetValue(CommandLineOptions.AOTCompilerModeOption), - WasmDataDirectory = parseResult.GetValue(CommandLineOptions.WasmDataDirectoryOption), - WasmCoreCLR = parseResult.GetValue(CommandLineOptions.WasmCoreCLROption), - NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), - NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), - Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), }; - if (invalidOptions.Any() || !Validate(options, NullLogger.Instance)) + if (!Validate(options, NullLogger.Instance)) { updatedArgs = null; return false; diff --git a/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs b/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs new file mode 100644 index 0000000000..a38bfd3b52 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs @@ -0,0 +1,33 @@ +using System; +using System.Globalization; + +namespace BenchmarkDotNet.Helpers +{ + internal static class CultureInfoHelper + { + public static IDisposable CreateInvariantUICultureScope() + => new InvariantUICultureScope(); + + private class InvariantUICultureScope : IDisposable + { + private readonly int savedThreadId; + private readonly CultureInfo savedCultureInfo; + + public InvariantUICultureScope() + { + savedThreadId = Environment.CurrentManagedThreadId; + savedCultureInfo = CultureInfo.CurrentUICulture; + + CultureInfo.CurrentUICulture = CultureInfo.InvariantCulture; + } + + public void Dispose() + { + if (Environment.CurrentManagedThreadId == savedThreadId) + { + CultureInfo.CurrentUICulture = savedCultureInfo; + } + } + } + } +} \ No newline at end of file From 30e73d420737e8df4cef2e7b8a5e4aecee4616be Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sun, 22 Feb 2026 14:07:41 +0200 Subject: [PATCH 14/18] refactor: remove implicit filters argument and handle unmatched tokens for filters --- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 2fdbbd8c41..d0a75077bf 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -76,7 +76,6 @@ public static class ConfigParser { "fullxml", new[] { XmlExporter.Full } } }; - internal static Argument ImplicitFiltersArgument { get; } = new Argument("implicitFilters") { Arity = ArgumentArity.ZeroOrMore }; private static RootCommand? _rootCommand; internal static RootCommand RootCommand @@ -158,7 +157,6 @@ internal static RootCommand RootCommand CommandLineOptions.NoEvaluationOverheadOption, CommandLineOptions.ResumeOption, }; - _rootCommand.Add(ImplicitFiltersArgument); } return _rootCommand; } @@ -388,7 +386,7 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) - .Concat(parseResult.GetValue(ImplicitFiltersArgument) ?? []) + .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-"))) .ToArray(), HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), @@ -595,7 +593,7 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act ExtraArguments = extraArgs, BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) - .Concat(parseResult.GetValue(ImplicitFiltersArgument) ?? []) + .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-"))) .ToArray(), }; From f4f3b326acbba3778c81292aa402de5ad45c798e Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sun, 22 Feb 2026 14:39:30 +0200 Subject: [PATCH 15/18] refactor: simplify RootCommand initialization and enhance command line options handling --- .../ConsoleArguments/ConfigParser.cs | 228 +++++++++++------- 1 file changed, 144 insertions(+), 84 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index d0a75077bf..e585be63e7 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -77,88 +77,83 @@ public static class ConfigParser }; - private static RootCommand? _rootCommand; internal static RootCommand RootCommand { get { - if (_rootCommand == null) - { - using var invariantUICultureScope = Helpers.CultureInfoHelper.CreateInvariantUICultureScope(); + using var invariantUICultureScope = Helpers.CultureInfoHelper.CreateInvariantUICultureScope(); - _rootCommand = new RootCommand("BenchmarkDotNet Command Line options") - { - CommandLineOptions.BaseJobOption, - CommandLineOptions.RuntimesOption, - CommandLineOptions.ExportersOption, - CommandLineOptions.MemoryOption, - CommandLineOptions.ThreadingOption, - CommandLineOptions.ExceptionsOption, - CommandLineOptions.DisassemblyOption, - CommandLineOptions.ProfilerOption, - CommandLineOptions.FiltersOption, - CommandLineOptions.HiddenColumnsOption, - CommandLineOptions.RunInProcessOption, - CommandLineOptions.ArtifactsDirectoryOption, - CommandLineOptions.OutliersOption, - CommandLineOptions.AffinityOption, - CommandLineOptions.DisplayAllStatisticsOption, - CommandLineOptions.AllCategoriesOption, - CommandLineOptions.AnyCategoriesOption, - CommandLineOptions.AttributeNamesOption, - CommandLineOptions.JoinOption, - CommandLineOptions.KeepBenchmarkFilesOption, - CommandLineOptions.DontOverwriteResultsOption, - CommandLineOptions.HardwareCountersOption, - CommandLineOptions.CliPathOption, - CommandLineOptions.RestorePathOption, - CommandLineOptions.CoreRunPathsOption, - CommandLineOptions.MonoPathOption, - CommandLineOptions.ClrVersionOption, - CommandLineOptions.ILCompilerVersionOption, - CommandLineOptions.IlcPackagesOption, - CommandLineOptions.LaunchCountOption, - CommandLineOptions.WarmupCountOption, - CommandLineOptions.MinWarmupCountOption, - CommandLineOptions.MaxWarmupCountOption, - CommandLineOptions.IterationTimeOption, - CommandLineOptions.IterationCountOption, - CommandLineOptions.MinIterationCountOption, - CommandLineOptions.MaxIterationCountOption, - CommandLineOptions.InvocationCountOption, - CommandLineOptions.UnrollFactorOption, - CommandLineOptions.RunStrategyOption, - CommandLineOptions.PlatformOption, - CommandLineOptions.RunOnceOption, - CommandLineOptions.PrintInformationOption, - CommandLineOptions.ApplesToApplesOption, - CommandLineOptions.ListBenchmarkCaseModeOption, - CommandLineOptions.DisassemblerDepthOption, - CommandLineOptions.DisassemblerFiltersOption, - CommandLineOptions.DisassemblerDiffOption, - CommandLineOptions.LogBuildOutputOption, - CommandLineOptions.GenerateBinLogOption, - CommandLineOptions.TimeoutOption, - CommandLineOptions.WakeLockOption, - CommandLineOptions.StopOnFirstErrorOption, - CommandLineOptions.StatisticalTestThresholdOption, - CommandLineOptions.DisableLogFileOption, - CommandLineOptions.MaxParameterColumnWidthOption, - CommandLineOptions.EnvironmentVariablesOption, - CommandLineOptions.MemoryRandomizationOption, - CommandLineOptions.WasmJavascriptEngineOption, - CommandLineOptions.WasmJavaScriptEngineArgumentsOption, - CommandLineOptions.CustomRuntimePackOption, - CommandLineOptions.AOTCompilerPathOption, - CommandLineOptions.AOTCompilerModeOption, - CommandLineOptions.WasmDataDirectoryOption, - CommandLineOptions.WasmCoreCLROption, - CommandLineOptions.NoForcedGCsOption, - CommandLineOptions.NoEvaluationOverheadOption, - CommandLineOptions.ResumeOption, - }; - } - return _rootCommand; + return new RootCommand("BenchmarkDotNet Command Line options") + { + CommandLineOptions.BaseJobOption, + CommandLineOptions.RuntimesOption, + CommandLineOptions.ExportersOption, + CommandLineOptions.MemoryOption, + CommandLineOptions.ThreadingOption, + CommandLineOptions.ExceptionsOption, + CommandLineOptions.DisassemblyOption, + CommandLineOptions.ProfilerOption, + CommandLineOptions.FiltersOption, + CommandLineOptions.HiddenColumnsOption, + CommandLineOptions.RunInProcessOption, + CommandLineOptions.ArtifactsDirectoryOption, + CommandLineOptions.OutliersOption, + CommandLineOptions.AffinityOption, + CommandLineOptions.DisplayAllStatisticsOption, + CommandLineOptions.AllCategoriesOption, + CommandLineOptions.AnyCategoriesOption, + CommandLineOptions.AttributeNamesOption, + CommandLineOptions.JoinOption, + CommandLineOptions.KeepBenchmarkFilesOption, + CommandLineOptions.DontOverwriteResultsOption, + CommandLineOptions.HardwareCountersOption, + CommandLineOptions.CliPathOption, + CommandLineOptions.RestorePathOption, + CommandLineOptions.CoreRunPathsOption, + CommandLineOptions.MonoPathOption, + CommandLineOptions.ClrVersionOption, + CommandLineOptions.ILCompilerVersionOption, + CommandLineOptions.IlcPackagesOption, + CommandLineOptions.LaunchCountOption, + CommandLineOptions.WarmupCountOption, + CommandLineOptions.MinWarmupCountOption, + CommandLineOptions.MaxWarmupCountOption, + CommandLineOptions.IterationTimeOption, + CommandLineOptions.IterationCountOption, + CommandLineOptions.MinIterationCountOption, + CommandLineOptions.MaxIterationCountOption, + CommandLineOptions.InvocationCountOption, + CommandLineOptions.UnrollFactorOption, + CommandLineOptions.RunStrategyOption, + CommandLineOptions.PlatformOption, + CommandLineOptions.RunOnceOption, + CommandLineOptions.PrintInformationOption, + CommandLineOptions.ApplesToApplesOption, + CommandLineOptions.ListBenchmarkCaseModeOption, + CommandLineOptions.DisassemblerDepthOption, + CommandLineOptions.DisassemblerFiltersOption, + CommandLineOptions.DisassemblerDiffOption, + CommandLineOptions.LogBuildOutputOption, + CommandLineOptions.GenerateBinLogOption, + CommandLineOptions.TimeoutOption, + CommandLineOptions.WakeLockOption, + CommandLineOptions.StopOnFirstErrorOption, + CommandLineOptions.StatisticalTestThresholdOption, + CommandLineOptions.DisableLogFileOption, + CommandLineOptions.MaxParameterColumnWidthOption, + CommandLineOptions.EnvironmentVariablesOption, + CommandLineOptions.MemoryRandomizationOption, + CommandLineOptions.WasmJavascriptEngineOption, + CommandLineOptions.WasmJavaScriptEngineArgumentsOption, + CommandLineOptions.CustomRuntimePackOption, + CommandLineOptions.AOTCompilerPathOption, + CommandLineOptions.AOTCompilerModeOption, + CommandLineOptions.WasmDataDirectoryOption, + CommandLineOptions.WasmCoreCLROption, + CommandLineOptions.NoForcedGCsOption, + CommandLineOptions.NoEvaluationOverheadOption, + CommandLineOptions.ResumeOption, + }; } } @@ -356,11 +351,14 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par foreach (var error in parseResult.Errors) { string msg = error.Message; + var badArg = args.FirstOrDefault(a => a.StartsWith("-") && msg.Contains(a)); + if (badArg != null) { msg = $"Option '{badArg.TrimStart('-')}' is unknown."; } + logger.WriteLineError(msg); } return (false, default, default); @@ -582,22 +580,84 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-")).ToList(); - if (parseResult.Errors.Any() || invalidOptions.Any()) - { - updatedArgs = null; - return false; - } - var options = new CommandLineOptions { ExtraArguments = extraArgs, BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", + Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], + Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], + UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), + UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), + UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), + UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), + Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-"))) .ToArray(), + HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], + RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), + ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), + Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), + Affinity = parseResult.GetValue(CommandLineOptions.AffinityOption), + DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), + AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? [], + AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? [], + AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? [], + Join = parseResult.GetValue(CommandLineOptions.JoinOption), + KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), + DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), + HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? [], + CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), + RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null + ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) + : null, + CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? [], + MonoPath = parseResult.GetValue(CommandLineOptions.MonoPathOption), + ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", + ILCompilerVersion = parseResult.GetValue(CommandLineOptions.ILCompilerVersionOption), + IlcPackages = parseResult.GetValue(CommandLineOptions.IlcPackagesOption), + LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), + WarmupIterationCount = parseResult.GetValue(CommandLineOptions.WarmupCountOption), + MinWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MinWarmupCountOption), + MaxWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MaxWarmupCountOption), + IterationTimeInMilliseconds = parseResult.GetValue(CommandLineOptions.IterationTimeOption), + IterationCount = parseResult.GetValue(CommandLineOptions.IterationCountOption), + MinIterationCount = parseResult.GetValue(CommandLineOptions.MinIterationCountOption), + MaxIterationCount = parseResult.GetValue(CommandLineOptions.MaxIterationCountOption), + InvocationCount = parseResult.GetValue(CommandLineOptions.InvocationCountOption), + UnrollFactor = parseResult.GetValue(CommandLineOptions.UnrollFactorOption), + RunStrategy = parseResult.GetValue(CommandLineOptions.RunStrategyOption), + Platform = parseResult.GetValue(CommandLineOptions.PlatformOption), + RunOncePerIteration = parseResult.GetValue(CommandLineOptions.RunOnceOption), + PrintInformation = parseResult.GetValue(CommandLineOptions.PrintInformationOption), + ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), + ListBenchmarkCaseMode = parseResult.GetValue(CommandLineOptions.ListBenchmarkCaseModeOption), + DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), + DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? [], + DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), + GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), + TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), + WakeLock = parseResult.GetValue(CommandLineOptions.WakeLockOption), + StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), + StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", + DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), + MaxParameterColumnWidth = parseResult.GetValue(CommandLineOptions.MaxParameterColumnWidthOption), + EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? [], + MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), + WasmJavascriptEngine = parseResult.GetValue(CommandLineOptions.WasmJavascriptEngineOption), + WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), + CustomRuntimePack = parseResult.GetValue(CommandLineOptions.CustomRuntimePackOption), + AOTCompilerPath = parseResult.GetValue(CommandLineOptions.AOTCompilerPathOption), + AOTCompilerMode = parseResult.GetValue(CommandLineOptions.AOTCompilerModeOption), + WasmDataDirectory = parseResult.GetValue(CommandLineOptions.WasmDataDirectoryOption), + WasmCoreCLR = parseResult.GetValue(CommandLineOptions.WasmCoreCLROption), + NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), + NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), + Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), }; - if (!Validate(options, NullLogger.Instance)) + if (invalidOptions.Any() || !Validate(options, NullLogger.Instance)) { updatedArgs = null; return false; From e093c650e006cee9b4789efe3a6ffc9472ec1ea0 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sun, 22 Feb 2026 15:13:25 +0200 Subject: [PATCH 16/18] refactor: streamline CultureInfoHelper class and improve scope management --- .../Helpers/CultureInfoHelper.cs | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs b/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs index a38bfd3b52..24ac368133 100644 --- a/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs +++ b/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs @@ -1,32 +1,31 @@ using System; using System.Globalization; -namespace BenchmarkDotNet.Helpers +namespace BenchmarkDotNet.Helpers; + +internal static class CultureInfoHelper { - internal static class CultureInfoHelper + public static IDisposable CreateInvariantUICultureScope() + => new InvariantUICultureScope(); + + private class InvariantUICultureScope : IDisposable { - public static IDisposable CreateInvariantUICultureScope() - => new InvariantUICultureScope(); + private readonly int savedThreadId; + private readonly CultureInfo savedCultureInfo; - private class InvariantUICultureScope : IDisposable + public InvariantUICultureScope() { - private readonly int savedThreadId; - private readonly CultureInfo savedCultureInfo; - - public InvariantUICultureScope() - { - savedThreadId = Environment.CurrentManagedThreadId; - savedCultureInfo = CultureInfo.CurrentUICulture; + savedThreadId = Environment.CurrentManagedThreadId; + savedCultureInfo = CultureInfo.CurrentUICulture; - CultureInfo.CurrentUICulture = CultureInfo.InvariantCulture; - } + CultureInfo.CurrentUICulture = CultureInfo.InvariantCulture; + } - public void Dispose() + public void Dispose() + { + if (Environment.CurrentManagedThreadId == savedThreadId) { - if (Environment.CurrentManagedThreadId == savedThreadId) - { - CultureInfo.CurrentUICulture = savedCultureInfo; - } + CultureInfo.CurrentUICulture = savedCultureInfo; } } } From 5e5be17fc4f9f39771ec288bbe25db3f4581c6e8 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sun, 22 Feb 2026 15:26:10 +0200 Subject: [PATCH 17/18] feat: enhance help and version output logging in ConfigParser --- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index e585be63e7..a11d93eeed 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -342,7 +342,9 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par if (args.Any(a => a == "-h" || a == "--help" || a == "-?" || a == "--version")) { using var invariantUICultureScope = Helpers.CultureInfoHelper.CreateInvariantUICultureScope(); - parseResult.Invoke(); + using var writer = new StringWriter(); + parseResult.Invoke(new InvocationConfiguration { Output = writer }); + logger.Write(writer.ToString()); return (false, default, default); } From fa51421fed68fdbe7970edaa098810f4e0438879 Mon Sep 17 00:00:00 2001 From: abdulrahmanhossam Date: Sun, 1 Mar 2026 21:57:58 +0200 Subject: [PATCH 18/18] Fix build errors: Migrate newly merged Wasm options to System.CommandLine architecture --- .../ConsoleArguments/CommandLineOptions.cs | 63 +++++++------------ .../ConsoleArguments/ConfigParser.cs | 29 +++++---- 2 files changed, 37 insertions(+), 55 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 27b35888f3..54c6160ab5 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -1,19 +1,16 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Toolchains.MonoAotLLVM; +using Perfolizer.Mathematics.OutlierDetection; using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.CommandLine; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Toolchains.MonoWasm; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Helpers; -using Perfolizer.Mathematics.OutlierDetection; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; namespace BenchmarkDotNet.ConsoleArguments { @@ -229,6 +226,13 @@ public class CommandLineOptions public static readonly Option ResumeOption = new("--resume") { Description = "Continue the execution if the last run was stopped." }; + public static readonly Option WasmRuntimeFlavorOption = new("--wasmRuntimeFlavor") + { Description = "Runtime flavor for WASM benchmarks: 'Mono' (default) uses the Mono runtime pack, 'CoreCLR' uses the CoreCLR runtime pack.", DefaultValueFactory = _ => Environments.RuntimeFlavor.Mono }; + + public static readonly Option WasmProcessTimeoutMinutesOption = new("--wasmProcessTimeout") + { Description = "Maximum time in minutes to wait for a single WASM benchmark process to finish before force killing it.", DefaultValueFactory = _ => 10 }; + + // Properties public string BaseJob { get; set; } = ""; public IEnumerable Runtimes { get; set; } = []; @@ -298,14 +302,9 @@ public bool UseDisassemblyDiagnoser public FileInfo? AOTCompilerPath { get; set; } public MonoAotCompilerMode AOTCompilerMode { get; set; } public DirectoryInfo? WasmDataDirectory { get; set; } - - [Option("wasmRuntimeFlavor", Required = false, Default = Environments.RuntimeFlavor.Mono, HelpText = "Runtime flavor for WASM benchmarks: 'Mono' (default) uses the Mono runtime pack, 'CoreCLR' uses the CoreCLR runtime pack.")] + public bool WasmCoreCLR { get; set; } public Environments.RuntimeFlavor WasmRuntimeFlavor { get; set; } - - [Option("wasmProcessTimeout", Required = false, Default = 10, HelpText = "Maximum time in minutes to wait for a single WASM benchmark process to finish before force killing it.")] public int WasmProcessTimeoutMinutes { get; set; } - - [Option("noForcedGCs", Required = false, HelpText = "Specifying would not forcefully induce any GCs.")] public bool NoForcedGCs { get; set; } public bool NoEvaluationOverhead { get; set; } public bool Resume { get; set; } @@ -316,7 +315,6 @@ public bool UseDisassemblyDiagnoser static CommandLineOptions() { - // Allow space-separated arrays (e.g. --exporters html rplot) RuntimesOption.AllowMultipleArgumentsPerToken = true; ExportersOption.AllowMultipleArgumentsPerToken = true; FiltersOption.AllowMultipleArgumentsPerToken = true; @@ -331,29 +329,11 @@ static CommandLineOptions() void AddUnrecognizedValidator(Option option) { - var shortName = new UnParserSettings { PreferShortName = true }; - var longName = new UnParserSettings { PreferShortName = false }; - - yield return new Example("Use Job.ShortRun for running the benchmarks", shortName, new CommandLineOptions { BaseJob = "short" }); - yield return new Example("Run benchmarks in process", shortName, new CommandLineOptions { RunInProcess = true }); - yield return new Example("Run benchmarks for .NET 4.7.2, .NET 8.0 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = ["net472", "net8.0", "Mono"] }); - yield return new Example("Run benchmarks for .NET Core 3.1, .NET 6.0 and .NET 8.0. .NET Core 3.1 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = ["netcoreapp3.1", "net6.0", "net8.0"] }); - yield return new Example("Use MemoryDiagnoser to get GC stats", shortName, new CommandLineOptions { UseMemoryDiagnoser = true }); - yield return new Example("Use DisassemblyDiagnoser to get disassembly", shortName, new CommandLineOptions { UseDisassemblyDiagnoser = true }); - yield return new Example("Use HardwareCountersDiagnoser to get hardware counter info", longName, new CommandLineOptions { HardwareCounters = [nameof(HardwareCounter.CacheMisses), nameof(HardwareCounter.InstructionRetired)] }); - yield return new Example("Run all benchmarks exactly once", shortName, new CommandLineOptions { BaseJob = "Dry", Filters = [Escape("*")] }); - yield return new Example("Run all benchmarks from System.Memory namespace", shortName, new CommandLineOptions { Filters = [Escape("System.Memory*")] }); - yield return new Example("Run all benchmarks from ClassA and ClassB using type names", shortName, new CommandLineOptions { Filters = ["ClassA", "ClassB"] }); - yield return new Example("Run all benchmarks from ClassA and ClassB using patterns", shortName, new CommandLineOptions { Filters = [Escape("*.ClassA.*"), Escape("*.ClassB.*")] }); - yield return new Example("Run all benchmarks called `BenchmarkName` and show the results in single summary", longName, new CommandLineOptions { Join = true, Filters = [Escape("*.BenchmarkName")] }); - yield return new Example("Run selected benchmarks once per iteration", longName, new CommandLineOptions { RunOncePerIteration = true }); - yield return new Example("Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations", longName, new CommandLineOptions { InvocationCount = 100, WarmupIterationCount = 1, IterationCount = 5}); - yield return new Example("Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations", longName, new CommandLineOptions { IterationTimeInMilliseconds = 250, MinIterationCount = 9, MaxIterationCount = 15}); - yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET Core 6.0 will be baseline because it was provided as first.", longName, - new CommandLineOptions { Filters = ["*"], Runtimes = ["net6.0", "net8.0"], StatisticalTestThreshold = "5%" }); - yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, - new CommandLineOptions { EnvironmentVariables = ["ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2"] }); - yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = ["Mean", "Ratio"], }); + option.Validators.Add(result => + { + foreach (var token in result.Tokens.Where(t => t.Value.StartsWith("-", StringComparison.Ordinal))) + result.AddError($"Unrecognized option: {token.Value}"); + }); } AddUnrecognizedValidator(RuntimesOption); @@ -368,6 +348,5 @@ void AddUnrecognizedValidator(Option option) AddUnrecognizedValidator(DisassemblerFiltersOption); AddUnrecognizedValidator(CoreRunPathsOption); } - } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 2e110d4c93..76c8bb7301 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -1,11 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; @@ -18,22 +13,24 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Toolchains.R2R; using BenchmarkDotNet.Toolchains.CoreRun; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.Mono; using BenchmarkDotNet.Toolchains.MonoAotLLVM; using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Toolchains.R2R; using Perfolizer.Horology; using Perfolizer.Mathematics.OutlierDetection; -using BenchmarkDotNet.Toolchains.Mono; using Perfolizer.Metrology; +using System; +using System.Collections.Generic; using System.CommandLine; -using System.CommandLine.Parsing; -using System.Runtime.InteropServices; -using System.CommandLine.Invocation; -using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Text; using RuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; namespace BenchmarkDotNet.ConsoleArguments @@ -153,6 +150,8 @@ internal static RootCommand RootCommand CommandLineOptions.NoForcedGCsOption, CommandLineOptions.NoEvaluationOverheadOption, CommandLineOptions.ResumeOption, + CommandLineOptions.WasmRuntimeFlavorOption, + CommandLineOptions.WasmProcessTimeoutMinutesOption, }; } } @@ -449,6 +448,8 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), + WasmRuntimeFlavor = parseResult.GetValue(CommandLineOptions.WasmRuntimeFlavorOption), + WasmProcessTimeoutMinutes = parseResult.GetValue(CommandLineOptions.WasmProcessTimeoutMinutesOption), }; bool isSuccess = Validate(options, logger); @@ -657,6 +658,8 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), + WasmRuntimeFlavor = parseResult.GetValue(CommandLineOptions.WasmRuntimeFlavorOption), + WasmProcessTimeoutMinutes = parseResult.GetValue(CommandLineOptions.WasmProcessTimeoutMinutesOption), }; if (invalidOptions.Any() || !Validate(options, NullLogger.Instance))