WASM benchmark uses WebAssembly SDK and supports CoreClr runtime

eng/Versions.props

@@ -11,7 +11,7 @@
     <MicrosoftNETILLinkPackageVersion>10.0.0-rc.1.25555.107</MicrosoftNETILLinkPackageVersion>
     <SystemThreadingChannelsPackageVersion>10.0.0-rc.1.25555.107</SystemThreadingChannelsPackageVersion>
     <MicrosoftExtensionsLoggingPackageVersion>10.0.0-rc.1.25555.107</MicrosoftExtensionsLoggingPackageVersion>
-    <BenchmarkDotNetVersion>0.16.0-custom.20260127.101</BenchmarkDotNetVersion>
+    <BenchmarkDotNetVersion>0.16.0-nightly.20260226.451</BenchmarkDotNetVersion>
     <MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>10.0.0-rc.1.25555.107</MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>
     <MicrosoftDotNetXHarnessCLIVersion>11.0.0-prerelease.26064.3</MicrosoftDotNetXHarnessCLIVersion>
   </PropertyGroup>

eng/pipelines/runtime-wasm-perf-jobs.yml

@@ -130,3 +130,25 @@ jobs:
           performanceRepoAlias: ${{ parameters.performanceRepoAlias }}
           ${{ each parameter in parameters.jobParameters }}:
             ${{ parameter.key }}: ${{ parameter.value }}
+
+  # Run CoreCLR WASM microbenchmarks perf job
+  - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }}
+    parameters:
+      jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }}
+      buildConfig: release
+      runtimeFlavor: coreclr
+      platforms:
+      - linux_x64
+      jobParameters:
+        liveLibrariesBuildConfig: Release
+        runtimeType: wasm_coreclr
+        codeGenType: 'wasm'
+        runKind: micro
+        logicalMachine: 'perfviper'
+        javascriptEngine: 'v8'
+        additionalJobIdentifier: coreclr_v8
+        downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
+        runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }}
+        performanceRepoAlias: ${{ parameters.performanceRepoAlias }}
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}

eng/pipelines/templates/runtime-perf-job.yml

@@ -90,7 +90,7 @@ jobs:
         displayName: Add Properties To Pipeline Env
 
       - ${{ if eq(parameters.runtimeType, 'wasm') }}:
-        # Download wasm
+        # Download wasm (Mono)
         - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}:
           - template: /eng/pipelines/templates/download-artifact-step.yml
             parameters:
@@ -106,6 +106,25 @@ jobs:
               artifactName: BrowserWasm
               displayName: BrowserWasm
               ${{ insert }}: ${{ parameters.downloadSpecificBuild }}
+
+      - ${{ if eq(parameters.runtimeType, 'wasm_coreclr') }}:
+        # Download wasm (CoreCLR)
+        - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}:
+          - template: /eng/pipelines/templates/download-artifact-step.yml
+            parameters:
+              unpackFolder: $(librariesDownloadDir)/BrowserWasmCoreCLR
+              artifactFileName: BrowserWasmCoreCLR.tar.gz
+              artifactName: BrowserWasmCoreCLR
+              displayName: BrowserWasmCoreCLR
+        - ${{ if ne(parameters.downloadSpecificBuild.buildId, '') }}:
+          - template: /eng/pipelines/templates/download-specific-artifact-step.yml
+            parameters:
+              unpackFolder: $(librariesDownloadDir)/BrowserWasmCoreCLR
+              artifactFileName: BrowserWasmCoreCLR.tar.gz
+              artifactName: BrowserWasmCoreCLR
+              displayName: BrowserWasmCoreCLR
+              ${{ insert }}: ${{ parameters.downloadSpecificBuild }}
+
       - ${{ elseif and(eq(parameters.codeGenType, 'AOT'), not(eq(parameters.runtimeType, 'AndroidMono'))) }}:
         - template: /eng/pipelines/templates/download-artifact-step.yml
           parameters:

scripts/benchmarks_local.py

@@ -245,14 +245,6 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co
             copy_directory_contents(src_dir_dotnet_latest, dest_dir_wasm_dotnet)
             src_dir_built_nugets = os.path.join(repo_path, "artifacts", "packages", "Release", "Shipping") # Goal is to copy Microsoft.NET.Sdk.WebAssembly.Pack*, Microsoft.NETCore.App.Ref*, either need to do the shipping folder or glob
             copy_directory_contents(src_dir_built_nugets, dir_bin_wasm)
-            # browser folder was extracted from wasm folder here: https://github.com/dotnet/runtime/pull/95940, so we need to check both locations for which to use (Dec, 2023)
-            src_file_test_main = glob.glob(os.path.join(repo_path, "src", "mono", "*", "test-main.js"))[0]
-            dest_dir_wasm_data = os.path.join(dir_bin_wasm, "wasm-data")
-            dest_file_test_main = os.path.join(dest_dir_wasm_data, "test-main.js")
-            if not os.path.exists(dest_dir_wasm_data):
-                os.makedirs(dest_dir_wasm_data)
-            shutil.copy2(src_file_test_main, dest_file_test_main)
-
             # Store the artifact in the artifact storage path
             shutil.rmtree(artifact_wasm_wasm, ignore_errors=True)
             copy_directory_contents(dir_bin_wasm, artifact_wasm_wasm)
@@ -402,7 +394,6 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type:
             '--anyCategories', 'Libraries', 'Runtime',
             '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono',
             '--cli', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle", "dotnet", "dotnet"),
-            '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle", "wasm-data"),
             '--wasmEngine', parsed_args.wasm_engine_path,
             '--wasmArgs', '\" --expose_wasm --module\"',
             '--logBuildOutput',
@@ -418,7 +409,6 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type:
             '--anyCategories', 'Libraries', 'Runtime',
             '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono',
             '--cli', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet", "dotnet"),
-            '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "wasm-data"),
             '--wasmEngine', parsed_args.wasm_engine_path,
             '--wasmArgs', '\" --expose_wasm --module\"',
             '--aotcompilermode', 'wasm',

scripts/build_runtime_payload.py

@@ -20,6 +20,7 @@
     "build_mono_payload",
     "build_monoaot_payload",
     "build_wasm_payload",
+    "build_wasm_coreclr_payload",
 ]
 
 
@@ -299,27 +300,15 @@ def build_monoaot_payload(
 
 def build_wasm_payload(
     browser_wasm_archive_or_dir: str,
-    payload_parent_dir: str,  # wasm creates three payload directories
-    test_main_js_path: Optional[str] = None,
-    runtime_repo_dir: Optional[str] = None,
+    payload_parent_dir: str,  # wasm creates two payload directories
 ) -> None:
-    """Create the WASM payload directories (dotnet, built-nugets, wasm-data).
+    """Create the WASM payload directories (dotnet, built-nugets).
 
     The archive/directory layout is expected to contain a `staging/` folder with
-    `dotnet-latest` and `built-nugets` subfolders. We also copy the harness
-    `test-main.js` into `wasm-data/`.
+    `dotnet-latest` and `built-nugets` subfolders.
     """
-    if test_main_js_path is None:
-        if runtime_repo_dir is None:
-            raise Exception("Please provide a path to the test-main.js or runtime repository")
-        test_main_js_path = os.path.join(runtime_repo_dir, "src", "mono", "browser", "test-main.js")
-
-    if not os.path.exists(test_main_js_path):
-        raise Exception(f"test-main.js not found in expected location: {test_main_js_path}")
-
     wasm_dotnet_dir = os.path.join(payload_parent_dir, "dotnet")
     wasm_built_nugets_dir = os.path.join(payload_parent_dir, "built-nugets")
-    wasm_data_dir = os.path.join(payload_parent_dir, "wasm-data")
 
     extract_archive_or_copy(
         browser_wasm_archive_or_dir, wasm_dotnet_dir, prefix="staging/dotnet-latest/"
@@ -329,7 +318,57 @@ def build_wasm_payload(
         browser_wasm_archive_or_dir, wasm_built_nugets_dir, prefix="staging/built-nugets/"
     )
 
-    os.makedirs(wasm_data_dir, exist_ok=True)
-    shutil.copy(test_main_js_path, os.path.join(wasm_data_dir, "test-main.js"))
+    _set_permissions_recursive([wasm_dotnet_dir, wasm_built_nugets_dir], mode=0o664) # rw-rw-r--
+
 
-    _set_permissions_recursive([wasm_dotnet_dir, wasm_built_nugets_dir, wasm_data_dir], mode=0o664) # rw-rw-r--
+def build_wasm_coreclr_payload(
+    browser_wasm_coreclr_archive_or_dir: str,
+    payload_parent_dir: str,
+) -> None:
+    """Create a WASM CoreCLR-only payload (dotnet).
+
+    This is a self-contained payload for running CoreCLR WASM benchmarks without
+    requiring Mono artifacts. The archive/directory layout is expected to contain
+    a `staging/` folder with `dotnet-none` (SDK) and
+    `microsoft.netcore.app.runtime.browser-wasm` (CoreCLR runtime pack) subfolders.
+    """
+
+    wasm_dotnet_dir = os.path.join(payload_parent_dir, "dotnet")
+    wasm_built_nugets_dir = os.path.join(payload_parent_dir, "built-nugets")
+
+    # Extract the SDK from dotnet-none
+    extract_archive_or_copy(
+        browser_wasm_coreclr_archive_or_dir, wasm_dotnet_dir, prefix="staging/dotnet-none/"
+    )
+
+    # Extract built NuGet packages (WebAssembly SDK pack, ref pack)
+    extract_archive_or_copy(
+        browser_wasm_coreclr_archive_or_dir, wasm_built_nugets_dir, prefix="staging/built-nugets/"
+    )
+
+    # Determine version from the runtime pack directory structure
+    runtime_pack_src = os.path.join(
+        browser_wasm_coreclr_archive_or_dir if os.path.isdir(browser_wasm_coreclr_archive_or_dir) else "",
+        "staging", "microsoft.netcore.app.runtime.browser-wasm", "Release"
+    )
+    if os.path.isdir(runtime_pack_src):
+        # Get version from NuGet.config or infer from directory
+        # For now, read version from the SDK's Microsoft.NETCore.App.Ref pack
+        ref_pack_parent = os.path.join(wasm_dotnet_dir, "packs", "Microsoft.NETCore.App.Ref")
+        if os.path.isdir(ref_pack_parent):
+            versions = os.listdir(ref_pack_parent)
+            if versions:
+                pack_version = versions[0]
+                coreclr_pack_dest = os.path.join(
+                    wasm_dotnet_dir, "packs", "Microsoft.NETCore.App.Runtime.browser-wasm", pack_version
+                )
+                extract_archive_or_copy(
+                    browser_wasm_coreclr_archive_or_dir,
+                    coreclr_pack_dest,
+                    prefix="staging/microsoft.netcore.app.runtime.browser-wasm/Release/",
+                )
+                getLogger().info("Installed CoreCLR browser-wasm runtime pack version %s", pack_version)
+        else:
+            getLogger().warning("Microsoft.NETCore.App.Ref pack not found – cannot determine version")
+
+    _set_permissions_recursive([wasm_dotnet_dir, wasm_built_nugets_dir], mode=0o664)

scripts/micro_benchmarks.py

@@ -140,6 +140,15 @@ def __get_bdn_arguments(user_input: str) -> list[str]:
         help='Tests should be run with the wasm runtime'
     )
 
+    parser.add_argument(
+        '--wasm-runtime-flavor',
+        dest='wasm_runtime_flavor',
+        required=False,
+        default='Mono',
+        choices=['Mono', 'CoreCLR'],
+        help='Runtime flavor for WASM benchmarks: Mono (default) or CoreCLR'
+    )
+
     parser.add_argument(
         '--bdn-arguments',
         dest='bdn_arguments',
@@ -273,6 +282,8 @@ def __get_benchmarkdotnet_arguments(framework: str, args: Any) -> list[str]:
             run_args += ['--runtimes', 'wasmnet11_0']
         else:
             raise ArgumentTypeError('Framework {} is not supported for wasm'.format(framework))
+        if args.wasm_runtime_flavor != 'Mono':
+            run_args += ['--wasmRuntimeFlavor', args.wasm_runtime_flavor]
 
     # Increase default 2 min build timeout to accommodate slow (or even very slow) hardware
     if not args.bdn_arguments or '--buildTimeout' not in args.bdn_arguments:

scripts/run_performance_job.py

@@ -200,8 +200,8 @@ def get_pre_commands(
                     "sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates"
                 ]
 
-    # Set up everything needed for WASM runs
-    if runtime_type == "wasm":  
+    # Set up everything needed for WASM runs (both Mono and CoreCLR)
+    if runtime_type in ("wasm", "wasm_coreclr"):  
         if os_distro == "azurelinux":
             # Azure Linux uses tdnf package manager
             install_prerequisites += [
@@ -442,7 +442,7 @@ def get_bdn_arguments(
             "--wasmEngine", javascript_engine_path,
             f"\\\"--wasmArgs={' '.join(wasm_args)}\\\"",
             "--cli", "$HELIX_CORRELATION_PAYLOAD/dotnet/dotnet",
-            "--wasmDataDir", "$HELIX_CORRELATION_PAYLOAD/wasm-data"
+            "--wasmProcessTimeout", "20",
         ]
 
         if is_aot:
@@ -451,6 +451,22 @@ def get_bdn_arguments(
                 "--buildTimeout", "3600"
             ]
 
+    if runtime_type == "wasm_coreclr":
+        category_exclusions += ["NoWASM", "NoWasmCoreCLR", "NoMono"]
+
+        wasm_args = ["--expose_wasm"]
+        if javascript_engine == "v8":
+            wasm_args += ["--module"]
+
+        assert javascript_engine_path is not None
+        bdn_arguments += [
+            "--wasmEngine", javascript_engine_path,
+            f"\\\"--wasmArgs={' '.join(wasm_args)}\\\"",
+            "--cli", "$HELIX_CORRELATION_PAYLOAD/dotnet/dotnet",
+            "--buildTimeout", "1200",
+            "--wasmProcessTimeout", "20"
+        ]
+
     if category_exclusions:
         bdn_arguments += ["--category-exclusion-filter", *set(category_exclusions)]
 
@@ -509,6 +525,12 @@ def get_run_configurations(
         if javascript_engine == "javascriptcore":
             configurations["JSEngine"] = "javascriptcore"
 
+    if runtime_type == "wasm_coreclr":
+        configurations["CompilationMode"] = "wasm"
+        configurations["RuntimeType"] = str(runtime_flavor)
+        if is_aot:
+            configurations["AOT"] = "true"
+
     if pgo_run_type == "nodynamicpgo":
         configurations["PGOType"] = "nodynamicpgo"
 
@@ -560,7 +582,7 @@ def get_run_configurations(
 
     return configurations
 
-def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str):
+def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False):
     if os_group == "windows":
         work_item_command = [
             "python",
@@ -588,6 +610,8 @@ def get_work_item_command(os_group: str, target_csproj: str, architecture: str,
 
     if wasm:
         work_item_command += ["--run-isolated", "--wasm", "--dotnet-path", "$HELIX_CORRELATION_PAYLOAD/dotnet/"]
+        if wasm_coreclr:
+            work_item_command += ["--wasm-runtime-flavor", "CoreCLR"]
 
     work_item_command += ["--bdn-artifacts", bdn_artifacts_dir]
 
@@ -646,8 +670,9 @@ def run_performance_job(args: RunPerformanceJobArgs):
     is_mono = args.runtime_type == "mono"
     mono_aot = is_mono and is_aot
     mono_dotnet = is_mono and not is_aot
-    wasm = args.runtime_type == "wasm"
-    wasm_aot = wasm and is_aot
+    wasm_coreclr = args.runtime_type == "wasm_coreclr"
+    wasm = args.runtime_type == "wasm" or wasm_coreclr  # wasm_coreclr also uses wasm infrastructure
+    wasm_aot = wasm and is_aot and not wasm_coreclr
 
     working_dir = os.path.join(args.performance_repo_dir, "CorrelationStaging") # folder in which the payload and workitem directories will be made
     work_item_dir = os.path.join(working_dir, "workitem", "") # Folder in which the work item commands will be run in
@@ -779,14 +804,29 @@ def run_performance_job(args: RunPerformanceJobArgs):
             shutil.copytree(args.mono_dotnet_dir, mono_dotnet_path, dirs_exist_ok=True)
 
     v8_version = ""
-    if wasm:
+    if wasm_coreclr:
+        if args.libraries_download_dir is None:
+            raise Exception("Libraries not downloaded for wasm_coreclr runs")
+
+        getLogger().info("Building wasm_coreclr payload directory")
+        browser_wasm_coreclr_dir = os.path.join(args.libraries_download_dir, "BrowserWasmCoreCLR")
+        build_wasm_coreclr_payload(
+            browser_wasm_coreclr_dir,
+            payload_dir,
+        )
+
+    elif wasm:
         if args.libraries_download_dir is None:
             raise Exception("Libraries not downloaded for wasm runs")
 
         getLogger().info("Copying wasm bundle directory to payload directory")
         browser_wasm_dir = os.path.join(args.libraries_download_dir, "BrowserWasm")
-        build_wasm_payload(browser_wasm_dir, payload_dir, runtime_repo_dir=args.runtime_repo_dir)
+        build_wasm_payload(
+            browser_wasm_dir,
+            payload_dir,
+        )
 
+    if wasm:
         if args.javascript_engine == "v8":
             if args.browser_versions_props_path is None:
                 if args.runtime_repo_dir is None:
@@ -802,7 +842,7 @@ def run_performance_job(args: RunPerformanceJobArgs):
                         break
                 else:
                     raise Exception("Unable to find v8 version in BrowserVersions.props")
-            
+
             if args.javascript_engine_path is None:
                 args.javascript_engine_path = f"/home/helixbot/.jsvu/bin/v8-{v8_version}"
 
@@ -961,6 +1001,11 @@ def run_performance_job(args: RunPerformanceJobArgs):
     ci_setup_arguments.output_file = os.path.join(root_payload_dir, "machine-setup")
     if args.is_scenario:
         ci_setup_arguments.install_dir = os.path.join(payload_dir, "dotnet")
+    elif wasm_coreclr:
+        # For wasm_coreclr, we already have the SDK in the payload - skip downloading
+        wasm_dotnet_path = os.path.join(payload_dir, "dotnet")
+        ci_setup_arguments.dotnet_path = wasm_dotnet_path
+        ci_setup_arguments.install_dir = wasm_dotnet_path
     else:
         tools_dir = os.path.join(performance_payload_dir, "tools")
         ci_setup_arguments.install_dir = os.path.join(tools_dir, "dotnet", args.architecture)
@@ -1157,7 +1202,7 @@ def get_bdn_args_for_coreroot_dir(coreroot_dir: Optional[str]):
 
     def get_work_item_command_for_artifact_dir(artifact_dir: str):
         assert args.target_csproj is not None
-        return get_work_item_command(args.os_group, args.target_csproj, args.architecture, perf_lab_framework, args.internal, wasm, artifact_dir)
+        return get_work_item_command(args.os_group, args.target_csproj, args.architecture, perf_lab_framework, args.internal, wasm, artifact_dir, wasm_coreclr)
 
     work_item_command = get_work_item_command_for_artifact_dir(bdn_artifacts_directory)
     baseline_work_item_command = get_work_item_command_for_artifact_dir(bdn_baseline_artifacts_dir)

src/Directory.Build.props

@@ -15,6 +15,7 @@
     <!-- every warning is important, we want to enforce best practices here to keep high quality of the code and avoid common mistakes -->
     <NoWarn>$(NoWarn);NU1507</NoWarn> <!-- Darc does not seem to support auto updating of packageSourceMapping causing tool publishes to fail without manual updating of the NuGet.config with package mappings, disable this check so we don't have to manually update the source mappings. -->
     <NoWarn>$(NoWarn);NETSDK1138</NoWarn> <!-- Disable warning about EOL target frameworks as we target them to test them -->
+    <NoWarn>$(NoWarn);CS9057</NoWarn> <!-- Suppress analyzer version mismatch when BDN analyzers target a newer Roslyn than the SDK provides -->
     <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
     <WarningLevel>4</WarningLevel>
 

src/benchmarks/micro/Categories.cs

@@ -49,6 +49,7 @@ public static class Categories
         public const string NoInterpreter = "NoInterpreter";
         public const string NoMono = "NoMono";
         public const string NoAOT = "NoAOT";
+        public const string NoWasmCoreCLR = "NoWasmCoreCLR";
         public const string Regex = "Regex";
     }
 }

src/benchmarks/micro/MicroBenchmarks.Wasm.props

@@ -1,7 +1,5 @@
 <Project>
   <PropertyGroup>
-    <WasmGenerateRunV8Script>true</WasmGenerateRunV8Script>
-    <WasmMainJSPath>$(WasmDataDir)\test-main.js</WasmMainJSPath>
     <TrimMode>partial</TrimMode>
     <JsonSerializerIsReflectionEnabledByDefault>true</JsonSerializerIsReflectionEnabledByDefault>
     <WasmBuildNative>true</WasmBuildNative>