diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index 5f00d599ec..bf4a7758a0 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index a494f42d46..54c6160ab5 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -1,18 +1,16 @@ -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; using BenchmarkDotNet.Configs; using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; -using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using CommandLine; -using CommandLine.Text; -using JetBrains.Annotations; using Perfolizer.Mathematics.OutlierDetection; +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; namespace BenchmarkDotNet.ConsoleArguments { @@ -23,251 +21,332 @@ public class CommandLineOptions private const int DefaultDisassemblerRecursiveDepth = 1; private bool useDisassemblyDiagnoser; - [Option('j', "job", Required = false, Default = "Default", HelpText = "Dry/Short/Medium/Long or Default")] - public string BaseJob { get; set; } = ""; + // Options defined in the same order as the original CommandLineParser implementation + public static readonly Option BaseJobOption = new("--job", "-j") + { Description = "Dry/Short/Medium/Long or Default", DefaultValueFactory = _ => "Default" }; - [Option('r', "runtimes", Required = false, HelpText = "Full target framework moniker for .NET Core and .NET. For Mono just 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline!")] - public IEnumerable Runtimes { get; set; } = []; + public static readonly Option RuntimesOption = new("--runtimes", "-r") + { Description = "Full target framework moniker for .NET Core and .NET. For Mono just 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline!" }; - [Option('e', "exporters", Required = false, HelpText = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML")] - public IEnumerable Exporters { get; set; } = []; + public static readonly Option ExportersOption = new("--exporters", "-e") + { Description = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML" }; - [Option('m', "memory", Required = false, Default = false, HelpText = "Prints memory statistics")] - public bool UseMemoryDiagnoser { get; set; } + public static readonly Option MemoryOption = new("--memory", "-m") + { Description = "Prints memory statistics" }; - [Option('t', "threading", Required = false, Default = false, HelpText = "Prints threading statistics")] - public bool UseThreadingDiagnoser { get; set; } + public static readonly Option ThreadingOption = new("--threading", "-t") + { Description = "Prints threading statistics" }; - [Option("exceptions", Required = false, Default = false, HelpText = "Prints exception statistics")] - public bool UseExceptionDiagnoser { get; set; } + public static readonly Option ExceptionsOption = new("--exceptions") + { Description = "Prints exception statistics" }; + + public static readonly Option DisassemblyOption = new("--disasm", "-d") + { Description = "Gets disassembly of benchmarked code" }; + + public static readonly Option ProfilerOption = new("--profiler", "-p") + { Description = "Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory" }; + + public static readonly Option FiltersOption = new("--filter", "-f") + { Description = "Glob patterns" }; + + public static readonly Option HiddenColumnsOption = new("--hide", "-h") + { Description = "Hides columns by name" }; + + public static readonly Option RunInProcessOption = new("--inProcess", "-i") + { Description = "Run benchmarks in Process" }; + + public static readonly Option ArtifactsDirectoryOption = new("--artifacts", "-a") + { Description = "Valid path to accessible directory" }; + + public static readonly Option OutliersOption = new("--outliers") + { Description = "DontRemove/RemoveUpper/RemoveLower/RemoveAll", DefaultValueFactory = _ => OutlierMode.RemoveUpper }; + + public static readonly Option AffinityOption = new("--affinity") + { Description = "Affinity mask to set for the benchmark process" }; + + public static readonly Option DisplayAllStatisticsOption = new("--allStats") + { Description = "Displays all statistics (min, max & more)" }; + + public static readonly Option AllCategoriesOption = new("--allCategories") + { Description = "Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed" }; + + public static readonly Option AnyCategoriesOption = new("--anyCategories") + { Description = "Any Categories to run" }; + + public static readonly Option AttributeNamesOption = new("--attribute") + { Description = "Run all methods with given attribute (applied to class or method)" }; + + public static readonly Option JoinOption = new("--join") + { Description = "Prints single table with results for all benchmarks" }; + + public static readonly Option KeepBenchmarkFilesOption = new("--keepFiles") + { Description = "Determines if all auto-generated files should be kept or removed after running the benchmarks." }; + + public static readonly Option DontOverwriteResultsOption = new("--noOverwrite") + { Description = "Determines if the exported result files should not be overwritten (be default they are overwritten)." }; + + public static readonly Option HardwareCountersOption = new("--counters") + { Description = "Hardware Counters" }; + + public static readonly Option CliPathOption = new("--cli") + { Description = "Path to dotnet cli (optional)." }; + + public static readonly Option RestorePathOption = new("--packages") + { Description = "The directory to restore packages to (optional)." }; + + public static readonly Option CoreRunPathsOption = new("--coreRun") + { Description = "Path(s) to CoreRun (optional)." }; + + public static readonly Option MonoPathOption = new("--monoPath") + { Description = "Optional path to Mono which should be used for running benchmarks." }; + + public static readonly Option ClrVersionOption = new("--clrVersion") + { Description = "Optional version of private CLR build used as the value of COMPLUS_Version env var." }; + + public static readonly Option ILCompilerVersionOption = new("--ilCompilerVersion") + { Description = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with NativeAOT. Example: \"7.0.0-preview.3.22123.2\"" }; + + public static readonly Option IlcPackagesOption = new("--ilcPackages") + { Description = "Optional path to shipping packages produced by local dotnet/runtime build." }; + + public static readonly Option LaunchCountOption = new("--launchCount") + { Description = "How many times we should launch process with target benchmark. The default is 1." }; + + public static readonly Option WarmupCountOption = new("--warmupCount") + { Description = "How many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic." }; + + public static readonly Option MinWarmupCountOption = new("--minWarmupCount") + { Description = "Minimum count of warmup iterations that should be performed. The default is 6." }; + + public static readonly Option MaxWarmupCountOption = new("--maxWarmupCount") + { Description = "Maximum count of warmup iterations that should be performed. The default is 50." }; + + public static readonly Option IterationTimeOption = new("--iterationTime") + { Description = "Desired time of execution of an iteration in milliseconds. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default" }; + + public static readonly Option IterationCountOption = new("--iterationCount") + { Description = "How many target iterations should be performed. By default calculated by the heuristic." }; + + public static readonly Option MinIterationCountOption = new("--minIterationCount") + { Description = "Minimum number of iterations to run. The default is 15." }; + + public static readonly Option MaxIterationCountOption = new("--maxIterationCount") + { Description = "Maximum number of iterations to run. The default is 100." }; + + public static readonly Option InvocationCountOption = new("--invocationCount") + { Description = "Invocation count in a single iteration. By default calculated by the heuristic." }; + + public static readonly Option UnrollFactorOption = new("--unrollFactor") + { Description = "How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default" }; + + public static readonly Option RunStrategyOption = new("--strategy") + { Description = "The RunStrategy that should be used. Throughput/ColdStart/Monitoring." }; + + public static readonly Option PlatformOption = new("--platform") + { Description = "The Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64." }; + + public static readonly Option RunOnceOption = new("--runOncePerIteration") + { Description = "Run the benchmark exactly once per iteration." }; + + public static readonly Option PrintInformationOption = new("--info") + { Description = "Print environment information." }; + + public static readonly Option ApplesToApplesOption = new("--apples") + { Description = "Runs apples-to-apples comparison for specified Jobs." }; + + public static readonly Option ListBenchmarkCaseModeOption = new("--list") + { Description = "Prints all of the available benchmark names. Flat/Tree", DefaultValueFactory = _ => ListBenchmarkCaseMode.Disabled }; + + public static readonly Option DisassemblerDepthOption = new("--disasmDepth") + { Description = "Sets the recursive depth for the disassembler.", DefaultValueFactory = _ => DefaultDisassemblerRecursiveDepth }; + + public static readonly Option DisassemblerFiltersOption = new("--disasmFilter") + { Description = "Glob patterns applied to full method signatures by the disassembler." }; + + public static readonly Option DisassemblerDiffOption = new("--disasmDiff") + { Description = "Generates diff reports for the disassembler." }; + + public static readonly Option LogBuildOutputOption = new("--logBuildOutput") + { Description = "Log Build output." }; + + public static readonly Option GenerateBinLogOption = new("--generateBinLog") + { Description = "Generate msbuild binlog for builds" }; + + public static readonly Option TimeoutOption = new("--buildTimeout") + { Description = "Build timeout in seconds." }; + + public static readonly Option WakeLockOption = new("--wakeLock") + { Description = "Prevents the system from entering sleep or turning off the display. None/System/Display." }; + + public static readonly Option StopOnFirstErrorOption = new("--stopOnFirstError") + { Description = "Stop on first error." }; + + public static readonly Option StatisticalTestThresholdOption = new("--statisticalTest") + { Description = "Threshold for Mann-Whitney U Test. Examples: 5%, 10ms, 100ns, 1s" }; + + public static readonly Option DisableLogFileOption = new("--disableLogFile") + { Description = "Disables the logfile." }; + + public static readonly Option MaxParameterColumnWidthOption = new("--maxWidth") + { Description = "Max parameter column width, the default is 20." }; + + public static readonly Option EnvironmentVariablesOption = new("--envVars") + { Description = "Colon separated environment variables (key:value)" }; + + public static readonly Option MemoryRandomizationOption = new("--memoryRandomization") + { Description = "Specifies whether Engine should allocate some random-sized memory between iterations." }; + + public static readonly Option WasmJavascriptEngineOption = new("--wasmEngine") + { Description = "Full path to a java script engine used to run the benchmarks, used by Wasm toolchain." }; + + public static readonly Option WasmJavaScriptEngineArgumentsOption = new("--wasmArgs") + { Description = "Arguments for the javascript engine used by Wasm toolchain.", DefaultValueFactory = _ => "--expose_wasm" }; + + public static readonly Option CustomRuntimePackOption = new("--customRuntimePack") + { Description = "Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently." }; + + public static readonly Option AOTCompilerPathOption = new("--AOTCompilerPath") + { Description = "Path to Mono AOT compiler, used for MonoAotLLVM." }; + + public static readonly Option AOTCompilerModeOption = new("--AOTCompilerMode") + { Description = "Mono AOT compiler mode, either 'mini' or 'llvm'", DefaultValueFactory = _ => MonoAotCompilerMode.mini }; + + public static readonly Option WasmDataDirectoryOption = new("--wasmDataDir") + { Description = "Wasm data directory" }; + + public static readonly Option WasmCoreCLROption = new("--wasmCoreCLR") + { Description = "Use CoreCLR runtime pack instead of the Mono runtime pack for WASM benchmarks." }; + + public static readonly Option NoForcedGCsOption = new("--noForcedGCs") + { Description = "Specifying would not forcefully induce any GCs." }; + + public static readonly Option NoEvaluationOverheadOption = new("--noOverheadEvaluation") + { Description = "Specifying would not run the evaluation overhead iterations." }; + + public static readonly Option ResumeOption = new("--resume") + { Description = "Continue the execution if the last run was stopped." }; + + public static readonly Option WasmRuntimeFlavorOption = new("--wasmRuntimeFlavor") + { Description = "Runtime flavor for WASM benchmarks: 'Mono' (default) uses the Mono runtime pack, 'CoreCLR' uses the CoreCLR runtime pack.", DefaultValueFactory = _ => Environments.RuntimeFlavor.Mono }; - [Option('d', "disasm", Required = false, Default = false, HelpText = "Gets disassembly of benchmarked code")] + public static readonly Option WasmProcessTimeoutMinutesOption = new("--wasmProcessTimeout") + { Description = "Maximum time in minutes to wait for a single WASM benchmark process to finish before force killing it.", DefaultValueFactory = _ => 10 }; + + + // Properties + public string BaseJob { get; set; } = ""; + public IEnumerable Runtimes { get; set; } = []; + public IEnumerable Exporters { get; set; } = []; + public bool UseMemoryDiagnoser { get; set; } + public bool UseThreadingDiagnoser { get; set; } + public bool UseExceptionDiagnoser { get; set; } public bool UseDisassemblyDiagnoser { get => useDisassemblyDiagnoser || DisassemblerRecursiveDepth != DefaultDisassemblerRecursiveDepth || DisassemblerFilters.Any(); set => useDisassemblyDiagnoser = value; } - - [Option('p', "profiler", Required = false, HelpText = "Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory")] - public string? Profiler { get; set; } - - [Option('f', "filter", Required = false, HelpText = "Glob patterns")] + public string Profiler { get; set; } = ""; public IEnumerable Filters { get; set; } = []; - - [Option('h', "hide", Required = false, HelpText = "Hides columns by name")] public IEnumerable HiddenColumns { get; set; } = []; - - [Option('i', "inProcess", Required = false, Default = false, HelpText = "Run benchmarks in Process")] public bool RunInProcess { get; set; } - - [Option('a', "artifacts", Required = false, HelpText = "Valid path to accessible directory")] public DirectoryInfo? ArtifactsDirectory { get; set; } - - [Option("outliers", Required = false, Default = OutlierMode.RemoveUpper, HelpText = "DontRemove/RemoveUpper/RemoveLower/RemoveAll")] public OutlierMode Outliers { get; set; } - - [Option("affinity", Required = false, HelpText = "Affinity mask to set for the benchmark process")] public int? Affinity { get; set; } - - [Option("allStats", Required = false, Default = false, HelpText = "Displays all statistics (min, max & more)")] public bool DisplayAllStatistics { get; set; } - - [Option("allCategories", Required = false, HelpText = "Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed")] public IEnumerable AllCategories { get; set; } = []; - - [Option("anyCategories", Required = false, HelpText = "Any Categories to run")] public IEnumerable AnyCategories { get; set; } = []; - - [Option("attribute", Required = false, HelpText = "Run all methods with given attribute (applied to class or method)")] public IEnumerable AttributeNames { get; set; } = []; - - [Option("join", Required = false, Default = false, HelpText = "Prints single table with results for all benchmarks")] public bool Join { get; set; } - - [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")] public bool KeepBenchmarkFiles { get; set; } - - [Option("noOverwrite", Required = false, Default = false, HelpText = "Determines if the exported result files should not be overwritten (be default they are overwritten).")] public bool DontOverwriteResults { get; set; } - - [Option("counters", Required = false, HelpText = "Hardware Counters", Separator = '+')] public IEnumerable HardwareCounters { get; set; } = []; - - [Option("cli", Required = false, HelpText = "Path to dotnet cli (optional).")] public FileInfo? CliPath { get; set; } - - [Option("packages", Required = false, HelpText = "The directory to restore packages to (optional).")] public DirectoryInfo? RestorePath { get; set; } - - [Option("coreRun", Required = false, HelpText = "Path(s) to CoreRun (optional).")] public IReadOnlyList CoreRunPaths { get; set; } = []; - - [Option("monoPath", Required = false, HelpText = "Optional path to Mono which should be used for running benchmarks.")] public FileInfo? MonoPath { get; set; } - - [Option("clrVersion", Required = false, HelpText = "Optional version of private CLR build used as the value of COMPLUS_Version env var.")] - public string? ClrVersion { get; set; } - - [Option("ilCompilerVersion", Required = false, HelpText = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with NativeAOT. Example: \"7.0.0-preview.3.22123.2\"")] + public string ClrVersion { get; set; } = ""; public string? ILCompilerVersion { get; set; } - - [Option("ilcPackages", Required = false, HelpText = @"Optional path to shipping packages produced by local dotnet/runtime build. Example: 'D:\projects\runtime\artifacts\packages\Release\Shipping\'")] public DirectoryInfo? IlcPackages { get; set; } - - [Option("launchCount", Required = false, HelpText = "How many times we should launch process with target benchmark. The default is 1.")] public int? LaunchCount { get; set; } - - [Option("warmupCount", Required = false, HelpText = "How many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic.")] public int? WarmupIterationCount { get; set; } - - [Option("minWarmupCount", Required = false, HelpText = "Minimum count of warmup iterations that should be performed. The default is 6.")] public int? MinWarmupIterationCount { get; set; } - - [Option("maxWarmupCount", Required = false, HelpText = "Maximum count of warmup iterations that should be performed. The default is 50.")] public int? MaxWarmupIterationCount { get; set; } - - [Option("iterationTime", Required = false, HelpText = "Desired time of execution of an iteration in milliseconds. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default")] public int? IterationTimeInMilliseconds { get; set; } - - [Option("iterationCount", Required = false, HelpText = "How many target iterations should be performed. By default calculated by the heuristic.")] public int? IterationCount { get; set; } - - [Option("minIterationCount", Required = false, HelpText = "Minimum number of iterations to run. The default is 15.")] public int? MinIterationCount { get; set; } - - [Option("maxIterationCount", Required = false, HelpText = "Maximum number of iterations to run. The default is 100.")] public int? MaxIterationCount { get; set; } - - [Option("invocationCount", Required = false, HelpText = "Invocation count in a single iteration. By default calculated by the heuristic.")] public long? InvocationCount { get; set; } - - [Option("unrollFactor", Required = false, HelpText = "How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default")] public int? UnrollFactor { get; set; } - - [Option("strategy", Required = false, HelpText = "The RunStrategy that should be used. Throughput/ColdStart/Monitoring.")] public RunStrategy? RunStrategy { get; set; } - - [Option("platform", Required = false, HelpText = "The Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64.")] public Platform? Platform { get; set; } - - [Option("runOncePerIteration", Required = false, Default = false, HelpText = "Run the benchmark exactly once per iteration.")] public bool RunOncePerIteration { get; set; } - - [Option("info", Required = false, Default = false, HelpText = "Print environment information.")] public bool PrintInformation { get; set; } - - [Option("apples", Required = false, Default = false, HelpText = "Runs apples-to-apples comparison for specified Jobs.")] public bool ApplesToApples { get; set; } - - [Option("list", Required = false, Default = ListBenchmarkCaseMode.Disabled, HelpText = "Prints all of the available benchmark names. Flat/Tree")] public ListBenchmarkCaseMode ListBenchmarkCaseMode { get; set; } - - [Option("disasmDepth", Required = false, Default = DefaultDisassemblerRecursiveDepth, HelpText = "Sets the recursive depth for the disassembler.")] public int DisassemblerRecursiveDepth { get; set; } - - [Option("disasmFilter", Required = false, HelpText = "Glob patterns applied to full method signatures by the the disassembler.")] public IEnumerable DisassemblerFilters { get; set; } = []; - - [Option("disasmDiff", Required = false, Default = false, HelpText = "Generates diff reports for the disassembler.")] public bool DisassemblerDiff { get; set; } - - [Option("logBuildOutput", Required = false, HelpText = "Log Build output.")] public bool LogBuildOutput { get; set; } - - [Option("generateBinLog", Required = false, HelpText = "Generate msbuild binlog for builds")] public bool GenerateMSBuildBinLog { get; set; } - - [Option("buildTimeout", Required = false, HelpText = "Build timeout in seconds.")] public int? TimeOutInSeconds { get; set; } - - [Option("wakeLock", Required = false, HelpText = "Prevents the system from entering sleep or turning off the display. None/System/Display.")] public WakeLockType? WakeLock { get; set; } - - [Option("stopOnFirstError", Required = false, Default = false, HelpText = "Stop on first error.")] public bool StopOnFirstError { get; set; } - - [Option("statisticalTest", Required = false, HelpText = "Threshold for Mann–Whitney U Test. Examples: 5%, 10ms, 100ns, 1s")] - public string? StatisticalTestThreshold { get; set; } - - [Option("disableLogFile", Required = false, HelpText = "Disables the logfile.")] + public string StatisticalTestThreshold { get; set; } = ""; public bool DisableLogFile { get; set; } - - [Option("maxWidth", Required = false, HelpText = "Max parameter column width, the default is 20.")] public int? MaxParameterColumnWidth { get; set; } - - [Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")] public IEnumerable EnvironmentVariables { get; set; } = []; - - [Option("memoryRandomization", Required = false, HelpText = "Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration.")] public bool MemoryRandomization { get; set; } - - [Option("wasmEngine", Required = false, HelpText = "Full path to a java script engine used to run the benchmarks, used by Wasm toolchain.")] public FileInfo? WasmJavascriptEngine { get; set; } - - [Option("wasmArgs", Required = false, Default = "--expose_wasm", HelpText = "Arguments for the javascript engine used by Wasm toolchain.")] public string? WasmJavaScriptEngineArguments { get; set; } - - [Option("customRuntimePack", Required = false, HelpText = "Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently.")] public string? CustomRuntimePack { get; set; } - - [Option("AOTCompilerPath", Required = false, HelpText = "Path to Mono AOT compiler, used for MonoAotLLVM.")] public FileInfo? AOTCompilerPath { get; set; } - - [Option("AOTCompilerMode", Required = false, Default = MonoAotCompilerMode.mini, HelpText = "Mono AOT compiler mode, either 'mini' or 'llvm'")] public MonoAotCompilerMode AOTCompilerMode { get; set; } - - [Option("wasmDataDir", Required = false, HelpText = "Wasm data directory")] public DirectoryInfo? WasmDataDirectory { get; set; } - - [Option("wasmRuntimeFlavor", Required = false, Default = Environments.RuntimeFlavor.Mono, HelpText = "Runtime flavor for WASM benchmarks: 'Mono' (default) uses the Mono runtime pack, 'CoreCLR' uses the CoreCLR runtime pack.")] + public bool WasmCoreCLR { get; set; } public Environments.RuntimeFlavor WasmRuntimeFlavor { get; set; } - - [Option("wasmProcessTimeout", Required = false, Default = 10, HelpText = "Maximum time in minutes to wait for a single WASM benchmark process to finish before force killing it.")] public int WasmProcessTimeoutMinutes { get; set; } - - [Option("noForcedGCs", Required = false, HelpText = "Specifying would not forcefully induce any GCs.")] public bool NoForcedGCs { get; set; } - - [Option("noOverheadEvaluation", Required = false, HelpText = "Specifying would not run the evaluation overhead iterations.")] public bool NoEvaluationOverhead { get; set; } - - [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")] public bool Resume { get; set; } - + public string[] ExtraArguments { get; set; } = []; internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any(); - [Usage(ApplicationAlias = "")] - [PublicAPI] - public static IEnumerable Examples + private static string Escape(string input) => UserInteractionHelper.EscapeCommandExample(input); + + static CommandLineOptions() { - get + RuntimesOption.AllowMultipleArgumentsPerToken = true; + ExportersOption.AllowMultipleArgumentsPerToken = true; + FiltersOption.AllowMultipleArgumentsPerToken = true; + AllCategoriesOption.AllowMultipleArgumentsPerToken = true; + AnyCategoriesOption.AllowMultipleArgumentsPerToken = true; + AttributeNamesOption.AllowMultipleArgumentsPerToken = true; + HiddenColumnsOption.AllowMultipleArgumentsPerToken = true; + HardwareCountersOption.AllowMultipleArgumentsPerToken = true; + EnvironmentVariablesOption.AllowMultipleArgumentsPerToken = true; + DisassemblerFiltersOption.AllowMultipleArgumentsPerToken = true; + CoreRunPathsOption.AllowMultipleArgumentsPerToken = true; + + void AddUnrecognizedValidator(Option option) { - var shortName = new UnParserSettings { PreferShortName = true }; - var longName = new UnParserSettings { PreferShortName = false }; - - yield return new Example("Use Job.ShortRun for running the benchmarks", shortName, new CommandLineOptions { BaseJob = "short" }); - yield return new Example("Run benchmarks in process", shortName, new CommandLineOptions { RunInProcess = true }); - yield return new Example("Run benchmarks for .NET 4.7.2, .NET 8.0 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = ["net472", "net8.0", "Mono"] }); - yield return new Example("Run benchmarks for .NET Core 3.1, .NET 6.0 and .NET 8.0. .NET Core 3.1 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = ["netcoreapp3.1", "net6.0", "net8.0"] }); - yield return new Example("Use MemoryDiagnoser to get GC stats", shortName, new CommandLineOptions { UseMemoryDiagnoser = true }); - yield return new Example("Use DisassemblyDiagnoser to get disassembly", shortName, new CommandLineOptions { UseDisassemblyDiagnoser = true }); - yield return new Example("Use HardwareCountersDiagnoser to get hardware counter info", longName, new CommandLineOptions { HardwareCounters = [nameof(HardwareCounter.CacheMisses), nameof(HardwareCounter.InstructionRetired)] }); - yield return new Example("Run all benchmarks exactly once", shortName, new CommandLineOptions { BaseJob = "Dry", Filters = [Escape("*")] }); - yield return new Example("Run all benchmarks from System.Memory namespace", shortName, new CommandLineOptions { Filters = [Escape("System.Memory*")] }); - yield return new Example("Run all benchmarks from ClassA and ClassB using type names", shortName, new CommandLineOptions { Filters = ["ClassA", "ClassB"] }); - yield return new Example("Run all benchmarks from ClassA and ClassB using patterns", shortName, new CommandLineOptions { Filters = [Escape("*.ClassA.*"), Escape("*.ClassB.*")] }); - yield return new Example("Run all benchmarks called `BenchmarkName` and show the results in single summary", longName, new CommandLineOptions { Join = true, Filters = [Escape("*.BenchmarkName")] }); - yield return new Example("Run selected benchmarks once per iteration", longName, new CommandLineOptions { RunOncePerIteration = true }); - yield return new Example("Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations", longName, new CommandLineOptions { InvocationCount = 100, WarmupIterationCount = 1, IterationCount = 5}); - yield return new Example("Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations", longName, new CommandLineOptions { IterationTimeInMilliseconds = 250, MinIterationCount = 9, MaxIterationCount = 15}); - yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET Core 6.0 will be baseline because it was provided as first.", longName, - new CommandLineOptions { Filters = ["*"], Runtimes = ["net6.0", "net8.0"], StatisticalTestThreshold = "5%" }); - yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, - new CommandLineOptions { EnvironmentVariables = ["ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2"] }); - yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = ["Mean", "Ratio"], }); + option.Validators.Add(result => + { + foreach (var token in result.Tokens.Where(t => t.Value.StartsWith("-", StringComparison.Ordinal))) + result.AddError($"Unrecognized option: {token.Value}"); + }); } - } - private static string Escape(string input) => UserInteractionHelper.EscapeCommandExample(input); + AddUnrecognizedValidator(RuntimesOption); + AddUnrecognizedValidator(ExportersOption); + AddUnrecognizedValidator(FiltersOption); + AddUnrecognizedValidator(AllCategoriesOption); + AddUnrecognizedValidator(AnyCategoriesOption); + AddUnrecognizedValidator(AttributeNamesOption); + AddUnrecognizedValidator(HiddenColumnsOption); + AddUnrecognizedValidator(HardwareCountersOption); + AddUnrecognizedValidator(EnvironmentVariablesOption); + AddUnrecognizedValidator(DisassemblerFiltersOption); + AddUnrecognizedValidator(CoreRunPathsOption); + } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index a3eebd9fa8..76c8bb7301 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -1,11 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; @@ -18,18 +13,25 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Toolchains.R2R; using BenchmarkDotNet.Toolchains.CoreRun; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.Mono; using BenchmarkDotNet.Toolchains.MonoAotLLVM; using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.NativeAot; -using CommandLine; +using BenchmarkDotNet.Toolchains.R2R; using Perfolizer.Horology; using Perfolizer.Mathematics.OutlierDetection; -using BenchmarkDotNet.Toolchains.Mono; using Perfolizer.Metrology; +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Text; +using RuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; namespace BenchmarkDotNet.ConsoleArguments { @@ -61,7 +63,7 @@ public static class ConfigParser { "stackoverflow", new[] { MarkdownExporter.StackOverflow } }, { "github", new[] { MarkdownExporter.GitHub } }, { "plain", new[] { PlainExporter.Default } }, - { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, // R Plots depends on having the full measurements available + { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, { "json", new[] { JsonExporter.Default } }, { "briefjson", new[] { JsonExporter.Brief } }, { "fulljson", new[] { JsonExporter.Full } }, @@ -71,26 +73,389 @@ public static class ConfigParser { "fullxml", new[] { XmlExporter.Full } } }; + + internal static RootCommand RootCommand + { + get + { + using var invariantUICultureScope = Helpers.CultureInfoHelper.CreateInvariantUICultureScope(); + + return new RootCommand("BenchmarkDotNet Command Line options") + { + CommandLineOptions.BaseJobOption, + CommandLineOptions.RuntimesOption, + CommandLineOptions.ExportersOption, + CommandLineOptions.MemoryOption, + CommandLineOptions.ThreadingOption, + CommandLineOptions.ExceptionsOption, + CommandLineOptions.DisassemblyOption, + CommandLineOptions.ProfilerOption, + CommandLineOptions.FiltersOption, + CommandLineOptions.HiddenColumnsOption, + CommandLineOptions.RunInProcessOption, + CommandLineOptions.ArtifactsDirectoryOption, + CommandLineOptions.OutliersOption, + CommandLineOptions.AffinityOption, + CommandLineOptions.DisplayAllStatisticsOption, + CommandLineOptions.AllCategoriesOption, + CommandLineOptions.AnyCategoriesOption, + CommandLineOptions.AttributeNamesOption, + CommandLineOptions.JoinOption, + CommandLineOptions.KeepBenchmarkFilesOption, + CommandLineOptions.DontOverwriteResultsOption, + CommandLineOptions.HardwareCountersOption, + CommandLineOptions.CliPathOption, + CommandLineOptions.RestorePathOption, + CommandLineOptions.CoreRunPathsOption, + CommandLineOptions.MonoPathOption, + CommandLineOptions.ClrVersionOption, + CommandLineOptions.ILCompilerVersionOption, + CommandLineOptions.IlcPackagesOption, + CommandLineOptions.LaunchCountOption, + CommandLineOptions.WarmupCountOption, + CommandLineOptions.MinWarmupCountOption, + CommandLineOptions.MaxWarmupCountOption, + CommandLineOptions.IterationTimeOption, + CommandLineOptions.IterationCountOption, + CommandLineOptions.MinIterationCountOption, + CommandLineOptions.MaxIterationCountOption, + CommandLineOptions.InvocationCountOption, + CommandLineOptions.UnrollFactorOption, + CommandLineOptions.RunStrategyOption, + CommandLineOptions.PlatformOption, + CommandLineOptions.RunOnceOption, + CommandLineOptions.PrintInformationOption, + CommandLineOptions.ApplesToApplesOption, + CommandLineOptions.ListBenchmarkCaseModeOption, + CommandLineOptions.DisassemblerDepthOption, + CommandLineOptions.DisassemblerFiltersOption, + CommandLineOptions.DisassemblerDiffOption, + CommandLineOptions.LogBuildOutputOption, + CommandLineOptions.GenerateBinLogOption, + CommandLineOptions.TimeoutOption, + CommandLineOptions.WakeLockOption, + CommandLineOptions.StopOnFirstErrorOption, + CommandLineOptions.StatisticalTestThresholdOption, + CommandLineOptions.DisableLogFileOption, + CommandLineOptions.MaxParameterColumnWidthOption, + CommandLineOptions.EnvironmentVariablesOption, + CommandLineOptions.MemoryRandomizationOption, + CommandLineOptions.WasmJavascriptEngineOption, + CommandLineOptions.WasmJavaScriptEngineArgumentsOption, + CommandLineOptions.CustomRuntimePackOption, + CommandLineOptions.AOTCompilerPathOption, + CommandLineOptions.AOTCompilerModeOption, + CommandLineOptions.WasmDataDirectoryOption, + CommandLineOptions.WasmCoreCLROption, + CommandLineOptions.NoForcedGCsOption, + CommandLineOptions.NoEvaluationOverheadOption, + CommandLineOptions.ResumeOption, + CommandLineOptions.WasmRuntimeFlavorOption, + CommandLineOptions.WasmProcessTimeoutMinutesOption, + }; + } + } + + private static bool HasDuplicateOptions(string[] args) + { + var aliasToCanonical = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["-j"] = "--job", + ["-r"] = "--runtimes", + ["-e"] = "--exporters", + ["-m"] = "--memory", + ["-t"] = "--threading", + ["-d"] = "--disasm", + ["-p"] = "--profiler", + ["-f"] = "--filter", + ["-h"] = "--hide", + ["-i"] = "--inprocess", + ["-a"] = "--artifacts" + }; + + var options = args.Where(a => a.StartsWith("-") && a != "--") + .Select(a => a.Split('=')[0].ToLowerInvariant()) + .Select(a => aliasToCanonical.TryGetValue(a, out var c) ? c : a); + + return options.GroupBy(x => x).Any(g => g.Count() > 1); + } + + private static string[] NormalizeArgs(string[] args) + { + var aliasToCanonical = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["-j"] = "--job", + ["-r"] = "--runtimes", + ["-e"] = "--exporters", + ["-m"] = "--memory", + ["-t"] = "--threading", + ["-d"] = "--disasm", + ["-p"] = "--profiler", + ["-f"] = "--filter", + ["-h"] = "--hide", + ["-i"] = "--inProcess", + ["-a"] = "--artifacts", + + ["--job"] = "--job", + ["--runtimes"] = "--runtimes", + ["--exporters"] = "--exporters", + ["--memory"] = "--memory", + ["--threading"] = "--threading", + ["--exceptions"] = "--exceptions", + ["--disasm"] = "--disasm", + ["--profiler"] = "--profiler", + ["--filter"] = "--filter", + ["--hide"] = "--hide", + ["--inprocess"] = "--inProcess", + ["--artifacts"] = "--artifacts", + ["--outliers"] = "--outliers", + ["--affinity"] = "--affinity", + ["--allstats"] = "--allStats", + ["--allcategories"] = "--allCategories", + ["--anycategories"] = "--anyCategories", + ["--attribute"] = "--attribute", + ["--join"] = "--join", + ["--keepfiles"] = "--keepFiles", + ["--nooverwrite"] = "--noOverwrite", + ["--counters"] = "--counters", + ["--cli"] = "--cli", + ["--packages"] = "--packages", + ["--corerun"] = "--coreRun", + ["--monopath"] = "--monoPath", + ["--clrversion"] = "--clrVersion", + ["--ilcompilerversion"] = "--ilCompilerVersion", + ["--ilcpackages"] = "--ilcPackages", + ["--launchcount"] = "--launchCount", + ["--warmupcount"] = "--warmupCount", + ["--minwarmupcount"] = "--minWarmupCount", + ["--maxwarmupcount"] = "--maxWarmupCount", + ["--iterationtime"] = "--iterationTime", + ["--iterationcount"] = "--iterationCount", + ["--miniterationcount"] = "--minIterationCount", + ["--maxiterationcount"] = "--maxIterationCount", + ["--invocationcount"] = "--invocationCount", + ["--unrollfactor"] = "--unrollFactor", + ["--strategy"] = "--strategy", + ["--platform"] = "--platform", + ["--runOncePerIteration"] = "--runOncePerIteration", + ["--runoncperiteration"] = "--runOncePerIteration", + ["--info"] = "--info", + ["--apples"] = "--apples", + ["--list"] = "--list", + ["--disasmdepth"] = "--disasmDepth", + ["--disasmfilter"] = "--disasmFilter", + ["--disasmdiff"] = "--disasmDiff", + ["--logbuildoutput"] = "--logBuildOutput", + ["--generatebinlog"] = "--generateBinLog", + ["--buildtimeout"] = "--buildTimeout", + ["--wakelock"] = "--wakeLock", + ["--stoponfirsterror"] = "--stopOnFirstError", + ["--statisticaltest"] = "--statisticalTest", + ["--disablelogfile"] = "--disableLogFile", + ["--maxwidth"] = "--maxWidth", + ["--envvars"] = "--envVars", + ["--memoryrandomization"] = "--memoryRandomization", + ["--wasmengine"] = "--wasmEngine", + ["--wasmargs"] = "--wasmArgs", + ["--customruntimepack"] = "--customRuntimePack", + ["--aotcompilerpath"] = "--AOTCompilerPath", + ["--aotcompilermode"] = "--AOTCompilerMode", + ["--wasmdatadir"] = "--wasmDataDir", + ["--wasmcoreclr"] = "--wasmCoreCLR", + ["--noforcedgcs"] = "--noForcedGCs", + ["--nooverheadevaluation"] = "--noOverheadEvaluation", + ["--resume"] = "--resume", + }; + + var result = new List(); + for (int i = 0; i < args.Length; i++) + { + var arg = args[i]; + + if (arg == "--") + { + result.Add(arg); + for (int j = i + 1; j < args.Length; j++) + result.Add(args[j]); + break; + } + + if (arg.StartsWith("-")) + { + var eqIdx = arg.IndexOf('='); + string key = eqIdx >= 0 ? arg.Substring(0, eqIdx) : arg; + string? value = eqIdx >= 0 ? arg.Substring(eqIdx + 1) : null; + + if (aliasToCanonical.TryGetValue(key, out var canonical)) + key = canonical; + + if (key.Equals("--counters", StringComparison.OrdinalIgnoreCase)) + { + result.Add(key); + if (value != null) + { + result.AddRange(value.Split('+')); + } + else if (i + 1 < args.Length && !args[i + 1].StartsWith("-") && args[i + 1].Contains('+')) + { + i++; + result.AddRange(args[i].Split('+')); + } + continue; + } + + arg = value != null ? $"{key}={value}" : key; + } + + result.Add(arg); + } + + return result.ToArray(); + } + public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) { - (bool isSuccess, IConfig? config, CommandLineOptions? options) result = default; + args = args.Where(a => !string.IsNullOrWhiteSpace(a)).ToArray(); - var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); - if (!expandSuccess) + if (HasDuplicateOptions(args)) { + logger.WriteLineError("Duplicate options are not allowed."); return (false, default, default); } + var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); + if (!expandSuccess) return (false, default, default); args = expandedArgs; - using (var parser = CreateParser(logger)) + args = NormalizeArgs(args); + + string[] extraArgs = []; + var dashDashIndex = Array.IndexOf(args, "--"); + if (dashDashIndex >= 0) + { + extraArgs = args.Skip(dashDashIndex + 1).ToArray(); + args = args.Take(dashDashIndex).ToArray(); + } + + var parseResult = RootCommand.Parse(args); + + if (args.Any(a => a == "-h" || a == "--help" || a == "-?" || a == "--version")) + { + using var invariantUICultureScope = Helpers.CultureInfoHelper.CreateInvariantUICultureScope(); + using var writer = new StringWriter(); + parseResult.Invoke(new InvocationConfiguration { Output = writer }); + logger.Write(writer.ToString()); + return (false, default, default); + } + + if (parseResult.Errors.Any()) + { + foreach (var error in parseResult.Errors) + { + string msg = error.Message; + + var badArg = args.FirstOrDefault(a => a.StartsWith("-") && msg.Contains(a)); + + if (badArg != null) + { + msg = $"Option '{badArg.TrimStart('-')}' is unknown."; + } + + logger.WriteLineError(msg); + } + return (false, default, default); + } + + var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-")).ToList(); + if (invalidOptions.Any()) { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig, args), options) : (false, default, default)) - .WithNotParsed(errors => result = (false, default, default)); + foreach (var opt in invalidOptions) + logger.WriteLineError($"Option '{opt.TrimStart('-')}' is unknown."); + return (false, default, default); } - return result; + var options = new CommandLineOptions + { + ExtraArguments = extraArgs, + BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", + Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], + Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], + UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), + UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), + UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), + UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), + Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", + Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) + .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-"))) + .ToArray(), + HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], + RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), + ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), + Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), + Affinity = parseResult.GetValue(CommandLineOptions.AffinityOption), + DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), + AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? [], + AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? [], + AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? [], + Join = parseResult.GetValue(CommandLineOptions.JoinOption), + KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), + DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), + HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? [], + CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), + RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null + ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) + : null, + CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? [], + MonoPath = parseResult.GetValue(CommandLineOptions.MonoPathOption), + ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", + ILCompilerVersion = parseResult.GetValue(CommandLineOptions.ILCompilerVersionOption), + IlcPackages = parseResult.GetValue(CommandLineOptions.IlcPackagesOption), + LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), + WarmupIterationCount = parseResult.GetValue(CommandLineOptions.WarmupCountOption), + MinWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MinWarmupCountOption), + MaxWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MaxWarmupCountOption), + IterationTimeInMilliseconds = parseResult.GetValue(CommandLineOptions.IterationTimeOption), + IterationCount = parseResult.GetValue(CommandLineOptions.IterationCountOption), + MinIterationCount = parseResult.GetValue(CommandLineOptions.MinIterationCountOption), + MaxIterationCount = parseResult.GetValue(CommandLineOptions.MaxIterationCountOption), + InvocationCount = parseResult.GetValue(CommandLineOptions.InvocationCountOption), + UnrollFactor = parseResult.GetValue(CommandLineOptions.UnrollFactorOption), + RunStrategy = parseResult.GetValue(CommandLineOptions.RunStrategyOption), + Platform = parseResult.GetValue(CommandLineOptions.PlatformOption), + RunOncePerIteration = parseResult.GetValue(CommandLineOptions.RunOnceOption), + PrintInformation = parseResult.GetValue(CommandLineOptions.PrintInformationOption), + ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), + ListBenchmarkCaseMode = parseResult.GetValue(CommandLineOptions.ListBenchmarkCaseModeOption), + DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), + DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? [], + DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), + GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), + TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), + WakeLock = parseResult.GetValue(CommandLineOptions.WakeLockOption), + StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), + StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", + DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), + MaxParameterColumnWidth = parseResult.GetValue(CommandLineOptions.MaxParameterColumnWidthOption), + EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? [], + MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), + WasmJavascriptEngine = parseResult.GetValue(CommandLineOptions.WasmJavascriptEngineOption), + WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), + CustomRuntimePack = parseResult.GetValue(CommandLineOptions.CustomRuntimePackOption), + AOTCompilerPath = parseResult.GetValue(CommandLineOptions.AOTCompilerPathOption), + AOTCompilerMode = parseResult.GetValue(CommandLineOptions.AOTCompilerModeOption), + WasmDataDirectory = parseResult.GetValue(CommandLineOptions.WasmDataDirectoryOption), + WasmCoreCLR = parseResult.GetValue(CommandLineOptions.WasmCoreCLROption), + NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), + NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), + Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), + WasmRuntimeFlavor = parseResult.GetValue(CommandLineOptions.WasmRuntimeFlavorOption), + WasmProcessTimeoutMinutes = parseResult.GetValue(CommandLineOptions.WasmProcessTimeoutMinutesOption), + }; + + bool isSuccess = Validate(options, logger); + return isSuccess + ? (true, CreateConfig(options, globalConfig, args), options) + : (false, default, default); } private static (bool Success, string[] ExpandedTokens) ExpandResponseFile(string[] args, ILogger logger) @@ -196,39 +561,296 @@ string GetToken() internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) { - (bool isSuccess, CommandLineOptions? options) result = default; + args = args.Where(a => !string.IsNullOrWhiteSpace(a)).ToArray(); - ILogger logger = NullLogger.Instance; - using (var parser = CreateParser(logger)) + if (HasDuplicateOptions(args)) { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, options) : (false, default)) - .WithNotParsed(errors => result = (false, default)); + updatedArgs = null; + return false; + } - if (!result.isSuccess) - { - updatedArgs = null; - return false; - } + args = NormalizeArgs(args); + + string[] extraArgs = []; + var dashDashIndex = Array.IndexOf(args, "--"); + if (dashDashIndex >= 0) + { + extraArgs = args.Skip(dashDashIndex + 1).ToArray(); + args = args.Take(dashDashIndex).ToArray(); + } - updater(result.options!); + var parseResult = RootCommand.Parse(args); - updatedArgs = parser.FormatCommandLine(result.options, settings => settings.SkipDefault = true).Split(); - return true; + var invalidOptions = parseResult.UnmatchedTokens.Where(t => t.StartsWith("-")).ToList(); + + var options = new CommandLineOptions + { + ExtraArguments = extraArgs, + BaseJob = parseResult.GetValue(CommandLineOptions.BaseJobOption) ?? "", + Runtimes = parseResult.GetValue(CommandLineOptions.RuntimesOption) ?? [], + Exporters = parseResult.GetValue(CommandLineOptions.ExportersOption) ?? [], + UseMemoryDiagnoser = parseResult.GetValue(CommandLineOptions.MemoryOption), + UseThreadingDiagnoser = parseResult.GetValue(CommandLineOptions.ThreadingOption), + UseExceptionDiagnoser = parseResult.GetValue(CommandLineOptions.ExceptionsOption), + UseDisassemblyDiagnoser = parseResult.GetValue(CommandLineOptions.DisassemblyOption), + Profiler = parseResult.GetValue(CommandLineOptions.ProfilerOption) ?? "", + Filters = (parseResult.GetValue(CommandLineOptions.FiltersOption) ?? []) + .Concat(parseResult.UnmatchedTokens.Where(t => !t.StartsWith("-"))) + .ToArray(), + HiddenColumns = parseResult.GetValue(CommandLineOptions.HiddenColumnsOption) ?? [], + RunInProcess = parseResult.GetValue(CommandLineOptions.RunInProcessOption), + ArtifactsDirectory = parseResult.GetValue(CommandLineOptions.ArtifactsDirectoryOption), + Outliers = parseResult.GetValue(CommandLineOptions.OutliersOption), + Affinity = parseResult.GetValue(CommandLineOptions.AffinityOption), + DisplayAllStatistics = parseResult.GetValue(CommandLineOptions.DisplayAllStatisticsOption), + AllCategories = parseResult.GetValue(CommandLineOptions.AllCategoriesOption) ?? [], + AnyCategories = parseResult.GetValue(CommandLineOptions.AnyCategoriesOption) ?? [], + AttributeNames = parseResult.GetValue(CommandLineOptions.AttributeNamesOption) ?? [], + Join = parseResult.GetValue(CommandLineOptions.JoinOption), + KeepBenchmarkFiles = parseResult.GetValue(CommandLineOptions.KeepBenchmarkFilesOption), + DontOverwriteResults = parseResult.GetValue(CommandLineOptions.DontOverwriteResultsOption), + HardwareCounters = parseResult.GetValue(CommandLineOptions.HardwareCountersOption) ?? [], + CliPath = parseResult.GetValue(CommandLineOptions.CliPathOption), + RestorePath = parseResult.GetValue(CommandLineOptions.RestorePathOption) != null + ? new DirectoryInfo(parseResult.GetValue(CommandLineOptions.RestorePathOption)!.FullName) + : null, + CoreRunPaths = parseResult.GetValue(CommandLineOptions.CoreRunPathsOption) ?? [], + MonoPath = parseResult.GetValue(CommandLineOptions.MonoPathOption), + ClrVersion = parseResult.GetValue(CommandLineOptions.ClrVersionOption) ?? "", + ILCompilerVersion = parseResult.GetValue(CommandLineOptions.ILCompilerVersionOption), + IlcPackages = parseResult.GetValue(CommandLineOptions.IlcPackagesOption), + LaunchCount = parseResult.GetValue(CommandLineOptions.LaunchCountOption), + WarmupIterationCount = parseResult.GetValue(CommandLineOptions.WarmupCountOption), + MinWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MinWarmupCountOption), + MaxWarmupIterationCount = parseResult.GetValue(CommandLineOptions.MaxWarmupCountOption), + IterationTimeInMilliseconds = parseResult.GetValue(CommandLineOptions.IterationTimeOption), + IterationCount = parseResult.GetValue(CommandLineOptions.IterationCountOption), + MinIterationCount = parseResult.GetValue(CommandLineOptions.MinIterationCountOption), + MaxIterationCount = parseResult.GetValue(CommandLineOptions.MaxIterationCountOption), + InvocationCount = parseResult.GetValue(CommandLineOptions.InvocationCountOption), + UnrollFactor = parseResult.GetValue(CommandLineOptions.UnrollFactorOption), + RunStrategy = parseResult.GetValue(CommandLineOptions.RunStrategyOption), + Platform = parseResult.GetValue(CommandLineOptions.PlatformOption), + RunOncePerIteration = parseResult.GetValue(CommandLineOptions.RunOnceOption), + PrintInformation = parseResult.GetValue(CommandLineOptions.PrintInformationOption), + ApplesToApples = parseResult.GetValue(CommandLineOptions.ApplesToApplesOption), + ListBenchmarkCaseMode = parseResult.GetValue(CommandLineOptions.ListBenchmarkCaseModeOption), + DisassemblerRecursiveDepth = parseResult.GetValue(CommandLineOptions.DisassemblerDepthOption), + DisassemblerFilters = parseResult.GetValue(CommandLineOptions.DisassemblerFiltersOption) ?? [], + DisassemblerDiff = parseResult.GetValue(CommandLineOptions.DisassemblerDiffOption), + LogBuildOutput = parseResult.GetValue(CommandLineOptions.LogBuildOutputOption), + GenerateMSBuildBinLog = parseResult.GetValue(CommandLineOptions.GenerateBinLogOption), + TimeOutInSeconds = parseResult.GetValue(CommandLineOptions.TimeoutOption), + WakeLock = parseResult.GetValue(CommandLineOptions.WakeLockOption), + StopOnFirstError = parseResult.GetValue(CommandLineOptions.StopOnFirstErrorOption), + StatisticalTestThreshold = parseResult.GetValue(CommandLineOptions.StatisticalTestThresholdOption) ?? "", + DisableLogFile = parseResult.GetValue(CommandLineOptions.DisableLogFileOption), + MaxParameterColumnWidth = parseResult.GetValue(CommandLineOptions.MaxParameterColumnWidthOption), + EnvironmentVariables = parseResult.GetValue(CommandLineOptions.EnvironmentVariablesOption) ?? [], + MemoryRandomization = parseResult.GetValue(CommandLineOptions.MemoryRandomizationOption), + WasmJavascriptEngine = parseResult.GetValue(CommandLineOptions.WasmJavascriptEngineOption), + WasmJavaScriptEngineArguments = parseResult.GetValue(CommandLineOptions.WasmJavaScriptEngineArgumentsOption), + CustomRuntimePack = parseResult.GetValue(CommandLineOptions.CustomRuntimePackOption), + AOTCompilerPath = parseResult.GetValue(CommandLineOptions.AOTCompilerPathOption), + AOTCompilerMode = parseResult.GetValue(CommandLineOptions.AOTCompilerModeOption), + WasmDataDirectory = parseResult.GetValue(CommandLineOptions.WasmDataDirectoryOption), + WasmCoreCLR = parseResult.GetValue(CommandLineOptions.WasmCoreCLROption), + NoForcedGCs = parseResult.GetValue(CommandLineOptions.NoForcedGCsOption), + NoEvaluationOverhead = parseResult.GetValue(CommandLineOptions.NoEvaluationOverheadOption), + Resume = parseResult.GetValue(CommandLineOptions.ResumeOption), + WasmRuntimeFlavor = parseResult.GetValue(CommandLineOptions.WasmRuntimeFlavorOption), + WasmProcessTimeoutMinutes = parseResult.GetValue(CommandLineOptions.WasmProcessTimeoutMinutesOption), + }; + + if (invalidOptions.Any() || !Validate(options, NullLogger.Instance)) + { + updatedArgs = null; + return false; } + + updater(options); + + updatedArgs = SerializeToArgs(options); + return true; } - private static Parser CreateParser(ILogger logger) - => new Parser(settings => + private static string[] SerializeToArgs(CommandLineOptions options) + { + var result = new List(); + + if (options.Filters.Any()) + { + result.Add("--filter"); + result.AddRange(options.Filters); + } + + if (options.BaseJob.IsNotBlank() && !options.BaseJob.Equals("Default", StringComparison.OrdinalIgnoreCase)) { - settings.CaseInsensitiveEnumValues = true; - settings.CaseSensitive = false; - settings.EnableDashDash = true; - settings.IgnoreUnknownArguments = false; - settings.HelpWriter = new LoggerWrapper(logger); - settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); - }); + result.Add("--job"); + result.Add(options.BaseJob); + } + + if (options.UseMemoryDiagnoser) result.Add("--memory"); + if (options.UseThreadingDiagnoser) result.Add("--threading"); + if (options.UseExceptionDiagnoser) result.Add("--exceptions"); + if (options.UseDisassemblyDiagnoser) result.Add("--disasm"); + if (options.RunInProcess) result.Add("--inProcess"); + if (options.Join) result.Add("--join"); + if (options.KeepBenchmarkFiles) result.Add("--keepFiles"); + if (options.DontOverwriteResults) result.Add("--noOverwrite"); + if (options.DisplayAllStatistics) result.Add("--allStats"); + if (options.RunOncePerIteration) result.Add("--runOncePerIteration"); + if (options.PrintInformation) result.Add("--info"); + if (options.ApplesToApples) result.Add("--apples"); + if (options.LogBuildOutput) result.Add("--logBuildOutput"); + if (options.GenerateMSBuildBinLog) result.Add("--generateBinLog"); + if (options.StopOnFirstError) result.Add("--stopOnFirstError"); + if (options.DisableLogFile) result.Add("--disableLogFile"); + if (options.MemoryRandomization) result.Add("--memoryRandomization"); + if (options.DisassemblerDiff) result.Add("--disasmDiff"); + if (options.WasmCoreCLR) result.Add("--wasmCoreCLR"); + if (options.NoForcedGCs) result.Add("--noForcedGCs"); + if (options.NoEvaluationOverhead) result.Add("--noOverheadEvaluation"); + if (options.Resume) result.Add("--resume"); + + if (options.Runtimes.Any()) + { + result.Add("--runtimes"); + result.AddRange(options.Runtimes); + } + + if (options.Profiler.IsNotBlank()) + { result.Add("--profiler"); result.Add(options.Profiler); } + + if (options.ClrVersion.IsNotBlank()) + { result.Add("--clrVersion"); result.Add(options.ClrVersion); } + + if (options.StatisticalTestThreshold.IsNotBlank()) + { result.Add("--statisticalTest"); result.Add(options.StatisticalTestThreshold); } + + if (options.ILCompilerVersion.IsNotBlank()) + { result.Add("--ilCompilerVersion"); result.Add(options.ILCompilerVersion!); } + + if (options.CustomRuntimePack.IsNotBlank()) + { result.Add("--customRuntimePack"); result.Add(options.CustomRuntimePack!); } + + if (options.WasmJavaScriptEngineArguments != null + && options.WasmJavaScriptEngineArguments != "--expose_wasm") + { result.Add("--wasmArgs"); result.Add(options.WasmJavaScriptEngineArguments); } + + if (options.Exporters.Any()) + { result.Add("--exporters"); result.AddRange(options.Exporters); } + + if (options.HardwareCounters.Any()) + { result.Add("--counters"); result.AddRange(options.HardwareCounters); } + + if (options.AllCategories.Any()) + { result.Add("--allCategories"); result.AddRange(options.AllCategories); } + + if (options.AnyCategories.Any()) + { result.Add("--anyCategories"); result.AddRange(options.AnyCategories); } + + if (options.AttributeNames.Any()) + { result.Add("--attribute"); result.AddRange(options.AttributeNames); } + + if (options.HiddenColumns.Any()) + { result.Add("--hide"); result.AddRange(options.HiddenColumns); } + + if (options.DisassemblerFilters.Any()) + { result.Add("--disasmFilter"); result.AddRange(options.DisassemblerFilters); } + + if (options.EnvironmentVariables.Any()) + { result.Add("--envVars"); result.AddRange(options.EnvironmentVariables); } + + if (options.LaunchCount.HasValue) + { result.Add("--launchCount"); result.Add(options.LaunchCount.Value.ToString()); } + + if (options.WarmupIterationCount.HasValue) + { result.Add("--warmupCount"); result.Add(options.WarmupIterationCount.Value.ToString()); } + + if (options.MinWarmupIterationCount.HasValue) + { result.Add("--minWarmupCount"); result.Add(options.MinWarmupIterationCount.Value.ToString()); } + + if (options.MaxWarmupIterationCount.HasValue) + { result.Add("--maxWarmupCount"); result.Add(options.MaxWarmupIterationCount.Value.ToString()); } + + if (options.IterationTimeInMilliseconds.HasValue) + { result.Add("--iterationTime"); result.Add(options.IterationTimeInMilliseconds.Value.ToString()); } + + if (options.IterationCount.HasValue) + { result.Add("--iterationCount"); result.Add(options.IterationCount.Value.ToString()); } + + if (options.MinIterationCount.HasValue) + { result.Add("--minIterationCount"); result.Add(options.MinIterationCount.Value.ToString()); } + + if (options.MaxIterationCount.HasValue) + { result.Add("--maxIterationCount"); result.Add(options.MaxIterationCount.Value.ToString()); } + + if (options.InvocationCount.HasValue) + { result.Add("--invocationCount"); result.Add(options.InvocationCount.Value.ToString()); } + + if (options.UnrollFactor.HasValue) + { result.Add("--unrollFactor"); result.Add(options.UnrollFactor.Value.ToString()); } + + if (options.Affinity.HasValue) + { result.Add("--affinity"); result.Add(options.Affinity.Value.ToString()); } + + if (options.TimeOutInSeconds.HasValue) + { result.Add("--buildTimeout"); result.Add(options.TimeOutInSeconds.Value.ToString()); } + + if (options.MaxParameterColumnWidth.HasValue) + { result.Add("--maxWidth"); result.Add(options.MaxParameterColumnWidth.Value.ToString()); } + + if (options.DisassemblerRecursiveDepth != 1) + { result.Add("--disasmDepth"); result.Add(options.DisassemblerRecursiveDepth.ToString()); } + + if (options.Outliers != OutlierMode.RemoveUpper) + { result.Add("--outliers"); result.Add(options.Outliers.ToString()); } + + if (options.RunStrategy.HasValue) + { result.Add("--strategy"); result.Add(options.RunStrategy.Value.ToString()); } + + if (options.Platform.HasValue) + { result.Add("--platform"); result.Add(options.Platform.Value.ToString()); } + + if (options.WakeLock.HasValue) + { result.Add("--wakeLock"); result.Add(options.WakeLock.Value.ToString()); } + + if (options.ListBenchmarkCaseMode != ListBenchmarkCaseMode.Disabled) + { result.Add("--list"); result.Add(options.ListBenchmarkCaseMode.ToString()); } + + if (options.AOTCompilerMode != MonoAotCompilerMode.mini) + { result.Add("--AOTCompilerMode"); result.Add(options.AOTCompilerMode.ToString()); } + + if (options.ArtifactsDirectory != null) + { result.Add("--artifacts"); result.Add(options.ArtifactsDirectory.FullName); } + + if (options.CliPath != null) + { result.Add("--cli"); result.Add(options.CliPath.FullName); } + + if (options.RestorePath != null) + { result.Add("--packages"); result.Add(options.RestorePath.FullName); } + + if (options.MonoPath != null) + { result.Add("--monoPath"); result.Add(options.MonoPath.FullName); } + + if (options.IlcPackages != null) + { result.Add("--ilcPackages"); result.Add(options.IlcPackages.FullName); } + + if (options.WasmJavascriptEngine != null) + { result.Add("--wasmEngine"); result.Add(options.WasmJavascriptEngine.FullName); } + + if (options.WasmDataDirectory != null) + { result.Add("--wasmDataDir"); result.Add(options.WasmDataDirectory.FullName); } + + if (options.AOTCompilerPath != null) + { result.Add("--AOTCompilerPath"); result.Add(options.AOTCompilerPath.FullName); } + + if (options.CoreRunPaths.Any()) + { result.Add("--coreRun"); result.AddRange(options.CoreRunPaths.Select(p => p.FullName)); } + + return result.ToArray(); + } private static bool Validate(CommandLineOptions options, ILogger logger) { @@ -330,9 +952,9 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC var config = new ManualConfig(); var baseJob = GetBaseJob(options, globalConfig); - var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID + var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); if (expanded.Length > 1) - expanded[0] = expanded[0].AsBaseline(); // if the user provides multiple jobs, then the first one should be a baseline + expanded[0] = expanded[0].AsBaseline(); config.AddJob(expanded); if (config.GetJobs().IsEmpty() && baseJob != Job.Default) config.AddJob(baseJob); @@ -401,7 +1023,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC private static Job GetBaseJob(CommandLineOptions options, IConfig? globalConfig) { var baseJob = - globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) // global config might define single custom Default job + globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) ?? AvailableJobs[options.BaseJob.ToLowerInvariant()]; if (baseJob != Job.Dry && options.Outliers != OutlierMode.RemoveUpper) @@ -441,9 +1063,9 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig? globalConfig) if (options.NoForcedGCs) baseJob = baseJob.WithGcForce(false); if (options.NoEvaluationOverhead) -#pragma warning disable CS0618 // Type or member is obsolete +#pragma warning disable CS0618 baseJob = baseJob.WithEvaluateOverhead(false); -#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning restore CS0618 if (options.EnvironmentVariables.Any()) { @@ -454,12 +1076,12 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig? globalConfig) }).ToArray()); } - if (AvailableJobs.Values.Contains(baseJob)) // no custom settings + if (AvailableJobs.Values.Contains(baseJob)) return baseJob; return baseJob - .AsDefault(false) // after applying all settings from console args the base job is not default anymore - .AsMutator(); // we mark it as mutator so it will be applied to other jobs defined via attributes and merged later in GetRunnableJobs method + .AsDefault(false) + .AsMutator(); } private static IEnumerable Expand(Job baseJob, CommandLineOptions options, string[] args) @@ -471,7 +1093,7 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, else if (options.ClrVersion.IsNotBlank()) { var runtime = ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion); - yield return baseJob.WithRuntime(runtime).WithId(runtime.Name); // local builds of .NET Runtime + yield return baseJob.WithRuntime(runtime).WithId(runtime.Name); } else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty()) { @@ -479,28 +1101,26 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, } else { - // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job string? first = args.FirstOrDefault(arg => arg.Equals("--runtimes", StringComparison.OrdinalIgnoreCase) || arg.Equals("-r", StringComparison.OrdinalIgnoreCase) - || arg.Equals("--corerun", StringComparison.OrdinalIgnoreCase)); if (first is null || first.Equals("--corerun", StringComparison.OrdinalIgnoreCase)) { foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + yield return CreateCoreRunJob(baseJob, options, coreRunPath); - foreach (string runtime in options.Runtimes) // known runtimes + foreach (string runtime in options.Runtimes) yield return CreateJobForGivenRuntime(baseJob, runtime, options); } else { - foreach (string runtime in options.Runtimes) // known runtimes + foreach (string runtime in options.Runtimes) yield return CreateJobForGivenRuntime(baseJob, runtime, options); foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + yield return CreateCoreRunJob(baseJob, options, coreRunPath); } } } @@ -562,61 +1182,45 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.NativeAot60: return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); - case RuntimeMoniker.NativeAot70: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); - case RuntimeMoniker.NativeAot80: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); - case RuntimeMoniker.NativeAot90: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"); - case RuntimeMoniker.NativeAot10_0: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"); - case RuntimeMoniker.NativeAot11_0: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet11/nuget/v3/index.json"); case RuntimeMoniker.WasmNet80: return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); - case RuntimeMoniker.WasmNet90: return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); - case RuntimeMoniker.WasmNet10_0: return MakeWasmJob(baseJob, options, "net10.0", runtimeMoniker); - case RuntimeMoniker.WasmNet11_0: return MakeWasmJob(baseJob, options, "net11.0", runtimeMoniker); case RuntimeMoniker.MonoAOTLLVM: return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet60: return MakeMonoAOTLLVMJob(baseJob, options, "net6.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet70: return MakeMonoAOTLLVMJob(baseJob, options, "net7.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet80: return MakeMonoAOTLLVMJob(baseJob, options, "net8.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet90: return MakeMonoAOTLLVMJob(baseJob, options, "net9.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet10_0: return MakeMonoAOTLLVMJob(baseJob, options, "net10.0", runtimeMoniker); - case RuntimeMoniker.MonoAOTLLVMNet11_0: return MakeMonoAOTLLVMJob(baseJob, options, "net11.0", runtimeMoniker); case RuntimeMoniker.Mono60: return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); - case RuntimeMoniker.Mono70: return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); - case RuntimeMoniker.Mono80: return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); @@ -674,11 +1278,11 @@ private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, s moniker: moniker); var toolChain = MonoAotLLVMToolChain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: monoAotLLVMRuntime.Name, - options: options)); + new NetCoreAppSettings( + targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, + runtimeFrameworkVersion: "", + name: monoAotLLVMRuntime.Name, + options: options)); return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain).WithId(monoAotLLVMRuntime.Name); } @@ -686,11 +1290,11 @@ private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, s private static Job CreateR2RJob(Job baseJob, CommandLineOptions options, Runtime runtime) { var toolChain = R2RToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: runtime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: runtime.Name, - options: options)); + new NetCoreAppSettings( + targetFrameworkMoniker: runtime.MsBuildMoniker, + runtimeFrameworkVersion: "", + name: runtime.Name, + options: options)); return baseJob.WithRuntime(runtime).WithToolchain(toolChain).WithId(runtime.Name); } @@ -730,18 +1334,6 @@ private static IEnumerable GetFilters(CommandLineOptions options) yield return new AttributesFilter(options.AttributeNames.ToArray()); } - private static int GetMaximumDisplayWidth() - { - try - { - return Console.WindowWidth; - } - catch (IOException) - { - return MinimumDisplayWidth; - } - } - private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, FileInfo coreRunPath) => baseJob .WithToolchain(new CoreRunToolchain( @@ -750,7 +1342,7 @@ private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, Fil targetFrameworkMoniker: RuntimeInformation.IsNetCore ? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker - : CoreRuntime.Latest.MsBuildMoniker, // use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors + : CoreRuntime.Latest.MsBuildMoniker, customDotNetCliPath: options.CliPath, restorePath: options.RestorePath, displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath))); @@ -764,18 +1356,6 @@ private static Job CreateCoreJobWithCli(Job baseJob, CommandLineOptions options) name: RuntimeInformation.GetCurrentRuntime().Name, options: options))); - /// - /// we have a limited amount of space when printing the output to the console, so we try to keep things small and simple - /// - /// for following paths: - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe - /// - /// we get: - /// - /// \Core_Root\CoreRun.exe - /// \Core_Root_beforeMyChanges\CoreRun.exe - /// private static string GetCoreRunToolchainDisplayName(IReadOnlyList paths, FileInfo coreRunPath) { if (paths.Count <= 1) @@ -809,7 +1389,6 @@ internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) runtime = runtime.Substring(0, index); } - // Monikers older than Net 10 don't use any version delimiter, newer monikers use _ delimiter. if (Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker)) { return true; @@ -817,4 +1396,4 @@ internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) return Enum.TryParse(runtime.Replace('.', '_'), ignoreCase: true, out runtimeMoniker); } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs b/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs new file mode 100644 index 0000000000..24ac368133 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/CultureInfoHelper.cs @@ -0,0 +1,32 @@ +using System; +using System.Globalization; + +namespace BenchmarkDotNet.Helpers; + +internal static class CultureInfoHelper +{ + public static IDisposable CreateInvariantUICultureScope() + => new InvariantUICultureScope(); + + private class InvariantUICultureScope : IDisposable + { + private readonly int savedThreadId; + private readonly CultureInfo savedCultureInfo; + + public InvariantUICultureScope() + { + savedThreadId = Environment.CurrentManagedThreadId; + savedCultureInfo = CultureInfo.CurrentUICulture; + + CultureInfo.CurrentUICulture = CultureInfo.InvariantCulture; + } + + public void Dispose() + { + if (Environment.CurrentManagedThreadId == savedThreadId) + { + CultureInfo.CurrentUICulture = savedCultureInfo; + } + } + } +} \ No newline at end of file