Add a windows launcher program to replace the current .bat files

This is still just an experiment but eventually I hope to remove
the `.bat` files in favor of this executable.

Users who have issues can opt out via `EMCC_USE_BAT_FILES`.

There are several reasons to want to do this:

1. Batch files are notoriously painful to work with and mis-understood.
2. Should be faster (no need to launch cmd.exe).
3. Works around several known issues with .bat files including one that
   is known unsolvable one (see emcc.bat for more details)

Author: sbc100

.circleci/config.yml

@@ -503,7 +503,7 @@ commands:
             echo "-----"
             echo "Running browser tests (EMTEST_BROWSER=$EMTEST_BROWSER)"
             echo "-----"
-            test/runner.bat << parameters.test_targets >>
+            test/runner.exe << parameters.test_targets >>
       - upload-test-results
   test-sockets-chrome:
     description: "Runs emscripten sockets tests under chrome"
@@ -1251,6 +1251,17 @@ jobs:
     steps:
       - test-sockets-chrome
 
+  build-windows-launcher:
+    executor:
+      name: win/server-2022
+      shell: cmd.exe
+    steps:
+      - checkout
+      - run: call .circleci\setup_vs2022.bat && cd tools\pylauncher && call build.bat
+      - store_artifacts:
+          path: tools/pylauncher/pylauncher.exe
+          destination: pylauncher.exe
+
   # windows and mac do not have separate build and test jobs, as they only run
   # a limited set of tests; it is simpler and faster to do it all in one job.
   test-windows:
@@ -1277,6 +1288,10 @@ jobs:
       EMTEST_BROWSER: "0"
     steps:
       - checkout
+      - run:
+          name: Build launcher
+          command: call .circleci\setup_vs2019.bat && cd tools\pylauncher && call build.bat
+          shell: cmd.exe
       - run:
           name: Install packages
           command: |
@@ -1286,17 +1301,14 @@ jobs:
       - install-emsdk
       - pip-install:
           python: "$EMSDK_PYTHON"
-      # run-tests depends on the ./test/runner script which is normally only
-      # created on UNIX systems (windows uses runner.bat)
-      - run: $EMSDK_PYTHON tools/maint/create_entry_points.py --all
       - run:
           name: "crossplatform tests"
-          command: test/runner.bat --crossplatform-only
+          command: test/runner.exe --crossplatform-only
       - upload-test-results
       # Run a single websockify-based test to ensure it works on windows.
       - run:
           name: "sockets.test_nodejs_sockets_echo*"
-          command: "test/runner.bat sockets.test_nodejs_sockets_echo*"
+          command: "test/runner.exe sockets.test_nodejs_sockets_echo*"
       - upload-test-results
 
   test-mac-arm64:
@@ -1393,6 +1405,7 @@ workflows:
       - test-node-compat
       - test-windows
       - test-windows-browser-firefox
+      - build-windows-launcher
       - test-mac-arm64:
           requires:
             - build-linux

.circleci/setup_vs2019.bat

@@ -0,0 +1,4 @@
+echo "setting up x64 toolchain"
+call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64
+where cl.exe
+echo "done"

.circleci/setup_vs2022.bat

@@ -0,0 +1,4 @@
+echo "setting up x64 toolchain"
+call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x64
+where cl.exe
+echo "done"

.gitignore

@@ -39,6 +39,8 @@ coverage.xml
 !/tools/run_python.ps1
 !/tools/run_python_compiler.ps1
 
+/tools/maint/pylauncher/pylauncher.obj
+
 # Shell scripts (created by ./tools/maint/create_entry_points.py)
 em++
 emcc
@@ -88,3 +90,28 @@ emsymbolizer.bat
 test/runner.bat
 tools/file_packager.bat
 tools/webidl_binder.bat
+
+bootstrap.exe
+em++.exe
+emcc.exe
+em-config.exe
+emar.exe
+embuilder.exe
+emcmake.exe
+emconfigure.exe
+emdump.exe
+emdwp.exe
+emmake.exe
+emnm.exe
+empath-split.exe
+emprofile.exe
+emranlib.exe
+emrun.exe
+emscan-deps.exe
+emscons.exe
+emsize.exe
+emstrip.exe
+emsymbolizer.exe
+tools/file_packager.exe
+tools/webidl_binder.exe
+test/runner.exe

ChangeLog.md

@@ -21,6 +21,11 @@ See docs/process.md for more on how version tagging works.
 5.0.2 (in development)
 ----------------------
 - SDL2 port updated from 2.32.8 to 2.32.10. (#26298)
+- The Windows `.bat` files that were previosly used to luanch emscripten's
+  python programs (e.g. `emcc.bat`) were removed and replaced with `.exe`
+  launchers.  These should be faster and have fewer limitations.  If you have
+  issues with these you can opt out by setting `EMCC_USE_BAT_FILES` in the
+  environment and re-run `tools/maint/create_entry_points.py`. (#24858)
 - The remaining launcher scripts (e.g. `emcc.bat`) were removed from the git
   repository.  These scripts are created by the `./bootstrap` script which
   must be run before the toolchain is usable (for folks using a git checkout of

cmake/Modules/Platform/Emscripten.cmake

@@ -66,7 +66,15 @@ get_filename_component(EMSCRIPTEN_ROOT_PATH "${EMSCRIPTEN_ROOT_PATH}" ABSOLUTE)
 list(APPEND CMAKE_MODULE_PATH "${EMSCRIPTEN_ROOT_PATH}/cmake/Modules")
 
 if (CMAKE_HOST_WIN32)
-  set(EMCC_SUFFIX ".bat")
+  # We use windows executables these days rather than `.bat` files, but we
+  # still project a fallback of using `.bat` files in the interim
+  if ("${EMSCRIPTEN_ROOT_PATH}/emcc.exe")
+    set(EMCC_SUFFIX ".exe")
+  else()
+    set(USE_BAT_FILES)
+    set(EMCC_SUFFIX ".bat")
+  else()
+  endif()
 else()
   set(EMCC_SUFFIX "")
 endif()
@@ -268,12 +276,15 @@ endif()
 
 set(CMAKE_EXECUTABLE_SUFFIX ".js")
 
-set(CMAKE_C_USE_RESPONSE_FILE_FOR_LIBRARIES 1)
-set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_LIBRARIES 1)
-set(CMAKE_C_USE_RESPONSE_FILE_FOR_OBJECTS 1)
-set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS 1)
-set(CMAKE_C_USE_RESPONSE_FILE_FOR_INCLUDES 1)
-set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_INCLUDES 1)
+if (DEFINED USE_BAT_FILES)
+  # See https://github.com/emscripten-core/emscripten/issues/2386
+  set(CMAKE_C_USE_RESPONSE_FILE_FOR_LIBRARIES 1)
+  set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_LIBRARIES 1)
+  set(CMAKE_C_USE_RESPONSE_FILE_FOR_OBJECTS 1)
+  set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS 1)
+  set(CMAKE_C_USE_RESPONSE_FILE_FOR_INCLUDES 1)
+  set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_INCLUDES 1)
+endif()
 
 set(CMAKE_C_RESPONSE_FILE_LINK_FLAG "@")
 set(CMAKE_CXX_RESPONSE_FILE_LINK_FLAG "@")

tools/maint/create_entry_points.py

@@ -7,13 +7,23 @@
 """Tool for creating/maintaining the python launcher scripts for all the emscripten
 python tools.
 
-This tool makes copies or `run_python.sh/.bat` and `run_python_compiler.sh/.bat`
-script for each entry point. On UNIX we previously used symbolic links for
-simplicity but this breaks MINGW users on windows who want to use the shell script
-launcher but don't have symlink support.
+This tool makes copies of the launcher for UNIX and/or windows for each of the
+python entry points.
+
+For UNIX we use a `run_python.sh` script that will exex that python executable.
+
+For windows we use `launcher.exe` which is small C program that launches python.
+
+Hitorically we used `run_python.bat` on windows but found that it was a constant
+source of bugs, as well we bring slower than the dedicated launcher.exe.
+
+On UNIX we previously used symbolic links for simplicity but this breaks MINGW
+users on windows who want to use the shell script launcher but don't have
+symlink support.
 """
 
 import os
+import shutil
 import stat
 import sys
 
@@ -61,12 +71,20 @@
 }
 
 
+windows_exe = os.path.join(__rootdir__, 'tools/pylauncher/pylauncher.exe')
+
+
 def make_executable(filename):
   old_mode = stat.S_IMODE(os.stat(filename).st_mode)
   os.chmod(filename, old_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 
 
-def main(all_platforms):
+def maybe_remove(filename):
+  if os.path.exists(filename):
+    os.remove(filename)
+
+
+def main(all_platforms, use_bat_files):
   is_windows = sys.platform.startswith('win')
   is_msys2 = 'MSYSTEM' in os.environ
   do_unix = all_platforms or not is_windows or is_msys2
@@ -99,15 +117,23 @@ def generate_entry_points(cmd, path):
         make_executable(launcher)
 
       if do_windows:
-        with open(launcher + '.bat', 'w') as f:
-          f.write(bat_data)
-
-        with open(launcher + '.ps1', 'w') as f:
-          f.write(ps1_data)
+        maybe_remove(launcher + '.bat')
+        maybe_remove(launcher + '.ps1')
+        maybe_remove(launcher + '.exe')
+        # We use windows executables these days rather than `.bat` files, but for an
+        # interim period we still support the old `.bat` files via the `EMCC_USE_BAT_FILES`
+        # environment variable.
+        if use_bat_files:
+          with open(launcher + '.bat', 'w') as f:
+            f.write(bat_data)
+          with open(launcher + '.ps1', 'w') as f:
+            f.write(ps1_data)
+        else:
+          shutil.copyfile(windows_exe, launcher + '.exe')
 
   generate_entry_points(entry_points, os.path.join(__scriptdir__, 'run_python'))
   generate_entry_points(compiler_entry_points, os.path.join(__scriptdir__, 'run_python_compiler'))
 
 
 if __name__ == '__main__':
-  sys.exit(main('--all' in sys.argv))
+  sys.exit(main('--all' in sys.argv, '--bat-files' in sys.argv))

tools/pylauncher/README.md

@@ -0,0 +1,11 @@
+Windows Python Script Launcher
+==============================
+
+This directory contains a simple launcher program for windows which is used to
+execute the emscripten compiler entry points using the python interpreter.  It
+uses the its own name (the name of the currently running executable) to
+determine which python script to run and serves the same purpose as the
+``run_python.sh`` script does on non-windows platforms.
+
+We build this executable statically using ``/MT`` so that it is maximally
+portable.

tools/pylauncher/build.bat

@@ -0,0 +1,4 @@
+:: /MT :  Statically link to the C runtime library for max portability
+:: /O1 : Favor small code (optimization for size)
+
+cl pylauncher.c /Fe:pylauncher.exe /MT /O1

tools/pylauncher/build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+cd ${SCRIPT_DIR}
+
+x86_64-w64-mingw32-gcc -Werror -static-libgcc -static-libstdc++ -s -Os pylauncher.c -o pylauncher.exe -lshlwapi -lshell32