diff --git a/.circleci/config.yml b/.circleci/config.yml index 09089ddf9b584..906db8bcbb633 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -503,7 +503,7 @@ commands: echo "-----" echo "Running browser tests (EMTEST_BROWSER=$EMTEST_BROWSER)" echo "-----" - test/runner.bat << parameters.test_targets >> + test/runner.exe << parameters.test_targets >> - upload-test-results test-sockets-chrome: description: "Runs emscripten sockets tests under chrome" @@ -1251,6 +1251,17 @@ jobs: steps: - test-sockets-chrome + build-windows-launcher: + executor: + name: win/server-2022 + shell: cmd.exe + steps: + - checkout + - run: call .circleci\setup_vs2022.bat && cd tools\pylauncher && call build.bat + - store_artifacts: + path: tools/pylauncher/pylauncher.exe + destination: pylauncher.exe + # windows and mac do not have separate build and test jobs, as they only run # a limited set of tests; it is simpler and faster to do it all in one job. test-windows: @@ -1277,6 +1288,10 @@ jobs: EMTEST_BROWSER: "0" steps: - checkout + - run: + name: Build launcher + command: call .circleci\setup_vs2019.bat && cd tools\pylauncher && call build.bat + shell: cmd.exe - run: name: Install packages command: | @@ -1286,17 +1301,14 @@ jobs: - install-emsdk - pip-install: python: "$EMSDK_PYTHON" - # run-tests depends on the ./test/runner script which is normally only - # created on UNIX systems (windows uses runner.bat) - - run: $EMSDK_PYTHON tools/maint/create_entry_points.py --all - run: name: "crossplatform tests" - command: test/runner.bat --crossplatform-only + command: test/runner.exe --crossplatform-only - upload-test-results # Run a single websockify-based test to ensure it works on windows. - run: name: "sockets.test_nodejs_sockets_echo*" - command: "test/runner.bat sockets.test_nodejs_sockets_echo*" + command: "test/runner.exe sockets.test_nodejs_sockets_echo*" - upload-test-results test-mac-arm64: @@ -1393,6 +1405,7 @@ workflows: - test-node-compat - test-windows - test-windows-browser-firefox + - build-windows-launcher - test-mac-arm64: requires: - build-linux diff --git a/.circleci/setup_vs2019.bat b/.circleci/setup_vs2019.bat new file mode 100644 index 0000000000000..cf84fc8f8086a --- /dev/null +++ b/.circleci/setup_vs2019.bat @@ -0,0 +1,4 @@ +echo "setting up x64 toolchain" +call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 +where cl.exe +echo "done" diff --git a/.circleci/setup_vs2022.bat b/.circleci/setup_vs2022.bat new file mode 100644 index 0000000000000..096f1f27ed46d --- /dev/null +++ b/.circleci/setup_vs2022.bat @@ -0,0 +1,4 @@ +echo "setting up x64 toolchain" +call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x64 +where cl.exe +echo "done" diff --git a/.gitignore b/.gitignore index 7e291889892ca..d9e8de7b5925b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,8 @@ coverage.xml !/tools/run_python.ps1 !/tools/run_python_compiler.ps1 +/tools/maint/pylauncher/pylauncher.obj + # Shell scripts (created by ./tools/maint/create_entry_points.py) em++ emcc @@ -88,3 +90,28 @@ emsymbolizer.bat test/runner.bat tools/file_packager.bat tools/webidl_binder.bat + +bootstrap.exe +em++.exe +emcc.exe +em-config.exe +emar.exe +embuilder.exe +emcmake.exe +emconfigure.exe +emdump.exe +emdwp.exe +emmake.exe +emnm.exe +empath-split.exe +emprofile.exe +emranlib.exe +emrun.exe +emscan-deps.exe +emscons.exe +emsize.exe +emstrip.exe +emsymbolizer.exe +tools/file_packager.exe +tools/webidl_binder.exe +test/runner.exe diff --git a/ChangeLog.md b/ChangeLog.md index f5acd99a7dbae..4c31b32d79256 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -21,6 +21,11 @@ See docs/process.md for more on how version tagging works. 5.0.2 (in development) ---------------------- - SDL2 port updated from 2.32.8 to 2.32.10. (#26298) +- The Windows `.bat` files that were previosly used to luanch emscripten's + python programs (e.g. `emcc.bat`) were removed and replaced with `.exe` + launchers. These should be faster and have fewer limitations. If you have + issues with these you can opt out by setting `EMCC_USE_BAT_FILES` in the + environment and re-run `tools/maint/create_entry_points.py`. (#24858) - The remaining launcher scripts (e.g. `emcc.bat`) were removed from the git repository. These scripts are created by the `./bootstrap` script which must be run before the toolchain is usable (for folks using a git checkout of diff --git a/cmake/Modules/Platform/Emscripten.cmake b/cmake/Modules/Platform/Emscripten.cmake index 293aad9f76358..a95b13abe0484 100644 --- a/cmake/Modules/Platform/Emscripten.cmake +++ b/cmake/Modules/Platform/Emscripten.cmake @@ -66,7 +66,14 @@ get_filename_component(EMSCRIPTEN_ROOT_PATH "${EMSCRIPTEN_ROOT_PATH}" ABSOLUTE) list(APPEND CMAKE_MODULE_PATH "${EMSCRIPTEN_ROOT_PATH}/cmake/Modules") if (CMAKE_HOST_WIN32) - set(EMCC_SUFFIX ".bat") + # We use windows executables these days rather than `.bat` files, but we + # still support a fallback of using `.bat` files. + if (EXISTS "${EMSCRIPTEN_ROOT_PATH}/emcc.exe") + set(EMCC_SUFFIX ".exe") + else() + set(USE_BAT_FILES) + set(EMCC_SUFFIX ".bat") + endif() else() set(EMCC_SUFFIX "") endif() diff --git a/tools/maint/create_entry_points.py b/tools/maint/create_entry_points.py index b8af5b24ee2f4..dd019c1229813 100755 --- a/tools/maint/create_entry_points.py +++ b/tools/maint/create_entry_points.py @@ -7,13 +7,23 @@ """Tool for creating/maintaining the python launcher scripts for all the emscripten python tools. -This tool makes copies or `run_python.sh/.bat` and `run_python_compiler.sh/.bat` -script for each entry point. On UNIX we previously used symbolic links for -simplicity but this breaks MINGW users on windows who want to use the shell script -launcher but don't have symlink support. +This tool makes copies of the launcher for UNIX and/or windows for each of the +python entry points. + +For UNIX we use a `run_python.sh` script that will exex that python executable. + +For windows we use `launcher.exe` which is small C program that launches python. + +Hitorically we used `run_python.bat` on windows but found that it was a constant +source of bugs, as well we bring slower than the dedicated launcher.exe. + +On UNIX we previously used symbolic links for simplicity but this breaks MINGW +users on windows who want to use the shell script launcher but don't have +symlink support. """ import os +import shutil import stat import sys @@ -61,12 +71,20 @@ } +windows_exe = os.path.join(__rootdir__, 'tools/pylauncher/pylauncher.exe') + + def make_executable(filename): old_mode = stat.S_IMODE(os.stat(filename).st_mode) os.chmod(filename, old_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) -def main(all_platforms): +def maybe_remove(filename): + if os.path.exists(filename): + os.remove(filename) + + +def main(all_platforms, use_bat_files): is_windows = sys.platform.startswith('win') is_msys2 = 'MSYSTEM' in os.environ do_unix = all_platforms or not is_windows or is_msys2 @@ -99,15 +117,23 @@ def generate_entry_points(cmd, path): make_executable(launcher) if do_windows: - with open(launcher + '.bat', 'w') as f: - f.write(bat_data) - - with open(launcher + '.ps1', 'w') as f: - f.write(ps1_data) + maybe_remove(launcher + '.bat') + maybe_remove(launcher + '.ps1') + maybe_remove(launcher + '.exe') + # We use windows executables these days rather than `.bat` files, but + # for an interim period we still support the old `.bat` files via the + # the `--bat-files` flag. + if use_bat_files: + with open(launcher + '.bat', 'w') as f: + f.write(bat_data) + with open(launcher + '.ps1', 'w') as f: + f.write(ps1_data) + else: + shutil.copyfile(windows_exe, launcher + '.exe') generate_entry_points(entry_points, os.path.join(__scriptdir__, 'run_python')) generate_entry_points(compiler_entry_points, os.path.join(__scriptdir__, 'run_python_compiler')) if __name__ == '__main__': - sys.exit(main('--all' in sys.argv)) + sys.exit(main('--all' in sys.argv, '--bat-files' in sys.argv)) diff --git a/tools/pylauncher/README.md b/tools/pylauncher/README.md new file mode 100644 index 0000000000000..4fbdf89c446c0 --- /dev/null +++ b/tools/pylauncher/README.md @@ -0,0 +1,11 @@ +Windows Python Script Launcher +============================== + +This directory contains a simple launcher program for windows which is used to +execute the emscripten compiler entry points using the python interpreter. It +uses the its own name (the name of the currently running executable) to +determine which python script to run and serves the same purpose as the +``run_python.sh`` script does on non-windows platforms. + +We build this executable statically using ``/MT`` so that it is maximally +portable. diff --git a/tools/pylauncher/build.bat b/tools/pylauncher/build.bat new file mode 100644 index 0000000000000..269524b115ee3 --- /dev/null +++ b/tools/pylauncher/build.bat @@ -0,0 +1,4 @@ +:: /MT : Statically link to the C runtime library for max portability +:: /O1 : Favor small code (optimization for size) + +cl pylauncher.c /Fe:pylauncher.exe /MT /O1 diff --git a/tools/pylauncher/build.sh b/tools/pylauncher/build.sh new file mode 100755 index 0000000000000..e61ea55fedec4 --- /dev/null +++ b/tools/pylauncher/build.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +cd ${SCRIPT_DIR} + +x86_64-w64-mingw32-gcc -Werror -static-libgcc -static-libstdc++ -s -Os pylauncher.c -o pylauncher.exe -lshlwapi -lshell32 diff --git a/tools/pylauncher/pylauncher.c b/tools/pylauncher/pylauncher.c new file mode 100644 index 0000000000000..d31c58591c738 --- /dev/null +++ b/tools/pylauncher/pylauncher.c @@ -0,0 +1,220 @@ +/* + * Copyright 2025 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + * + * Small win32 application that is used to launcher emscripten via python.exe. + * On non-windows platforms this is done via the run_pyton.sh shell script. + * + * The binary will look for a python script that matches its own name and run + * that using python.exe. + */ + +// Define _WIN32_WINNT to Windows 7 for max portability +#define _WIN32_WINNT 0x0601 + +#include +#include +#include +#include +#include +#include + +#pragma comment(lib, "shlwapi.lib") +#pragma comment(lib, "shell32.lib") + +bool launcher_debug = false; + +int dbg(const char* format, ...) { + if (launcher_debug) { + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + } +} + +const wchar_t* get_python_executable() { + const wchar_t* python_exe_w = _wgetenv(L"EMSDK_PYTHON"); + if (!python_exe_w) { + return L"python.exe"; + } + return python_exe_w; +} + +// Get the name of the currently running executable (module) +wchar_t* get_module_path() { + DWORD buffer_size = MAX_PATH; + wchar_t* module_path_w = malloc(sizeof(wchar_t) * buffer_size); + if (!module_path_w) + abort(); + + DWORD path_len = GetModuleFileNameW(NULL, module_path_w, buffer_size); + while (path_len > 0 && path_len == buffer_size) { + buffer_size *= 2; + module_path_w = realloc(module_path_w, sizeof(wchar_t) * buffer_size); + if (!module_path_w) + abort(); + path_len = GetModuleFileNameW(NULL, module_path_w, buffer_size); + } + + if (path_len == 0) + abort(); + + return module_path_w; +} + +/** + * A custom replacement for PathGetArgsW that is safe for command lines + * longer than MAX_PATH. + */ +const wchar_t* find_args(const wchar_t* command_line) { + const wchar_t* p = command_line; + + // Skip past the executable name, which can be quoted. + if (*p == L'"') { + // The path is quoted, find the closing quote. + p++; + while (*p) { + if (*p == L'"') { + p++; + break; + } + p++; + } + } else { + // The path is not quoted, find the first space. + while (*p && *p != L' ' && *p != L'\t') { + p++; + } + } + + // Skip any whitespace between the executable and the first argument. + while (*p && (*p == L' ' || *p == L'\t')) { + p++; + } + + return p; +} + +/** + * Create the script path by taking the launcher path and replacing the + * extension with .py. For example `C:\path\to\emcc.exe` becomes + * `C:\path\to\emcc.py`. + * + * If the corresponging .py file does not exist then also look it in the tools + * subdirectory. e.g. `C:\path\to\tools\emcc.py` + */ +wchar_t* get_script_path(const wchar_t* launcher_path) { + wchar_t* script_path = wcsdup(launcher_path); + if (!script_path) + abort(); + PathRemoveExtensionW(script_path); + size_t current_len = wcslen(script_path); + // 4 for `.py` and the null terminator + size_t new_size_in_chars = current_len + 4; + script_path = realloc(script_path, new_size_in_chars * sizeof(wchar_t)); + if (!script_path) + abort(); + wcscat_s(script_path, new_size_in_chars, L".py"); + if (PathFileExistsW(script_path)) { + return script_path; + } + + // Python file not found alongside launcher; try under tools + // C:\path\to\emcc.py` => C:\path\to\tools\emcc.py` + wchar_t* script_path_copy = wcsdup(script_path); + wchar_t* basename = PathFindFileNameW(script_path_copy); + // We need to add 6 more chars for 'tools\'. + new_size_in_chars += 6; + wchar_t* script_path_tools = malloc(new_size_in_chars * sizeof(wchar_t)); + wcscpy(script_path_tools, script_path); + PathRemoveFileSpecW(script_path_tools); + PathCombineW(script_path_tools, script_path_tools, L"tools"); + PathCombineW(script_path_tools, script_path_tools, basename); + + if (!PathFileExistsW(script_path_tools)) { + fprintf(stderr, "pylauncher: target python file not found: %ls / %ls\n", script_path, script_path_tools); + abort(); + } + free(script_path_copy); + free(script_path); + + return script_path_tools; +} + +int main(int argc, char* argv[]) { + // Setting EMCC_LAUNCHER_DEBUG enabled debug output for the launcher itself. + launcher_debug = GetEnvironmentVariableW(L"EMCC_LAUNCHER_DEBUG", NULL, 0); + + dbg("pylauncher: main\n"); + + const wchar_t* ccache = NULL; + DWORD env_len = GetEnvironmentVariableW(L"_EMCC_CCACHE", NULL, 0); + if (env_len) { + dbg("pylauncher: running via ccache.exe\n"); + ccache = L"ccache.exe"; + SetEnvironmentVariableW(L"_EMCC_CCACHE", NULL); + } + + const wchar_t* application_name = get_python_executable(); + wchar_t* launcher_path_w = get_module_path(); + wchar_t* script_path_w = get_script_path(launcher_path_w); + size_t command_line_len = wcslen(application_name) + wcslen(script_path_w) + 9; + if (ccache) { + command_line_len += wcslen(ccache) + 1; + } + wchar_t* command_line = malloc(sizeof(wchar_t) * command_line_len); + if (ccache) { + swprintf(command_line, command_line_len, L"%ls \"%ls\" -E \"%ls\"", ccache, application_name, script_path_w); + } else { + swprintf(command_line, command_line_len, L"\"%ls\" -E \"%ls\"", application_name, script_path_w); + } + + // -E will not ignore _PYTHON_SYSCONFIGDATA_NAME an internal + // of cpython used in cross compilation via setup.py. + SetEnvironmentVariableW(L"_PYTHON_SYSCONFIGDATA_NAME", NULL); + + // Build the final command line by appending the original arguments + const wchar_t* all_args = find_args(GetCommandLineW()); + if (all_args && *all_args) { + size_t current_len = wcslen(command_line); + size_t args_len = wcslen(all_args); + // +2 for the space and the null terminator + command_line = realloc(command_line, (current_len + args_len + 2) * sizeof(wchar_t)); + if (!command_line) + abort(); + wcscat_s(command_line, current_len + args_len + 2, L" "); + wcscat_s(command_line, current_len + args_len + 2, all_args); + } + + // Work around python bug 34780 by closing stdin, so that it is not inherited + // by the python subprocess. + env_len = GetEnvironmentVariableW(L"EM_WORKAROUND_PYTHON_BUG_34780", NULL, 0); + if (env_len) { + dbg("pylauncher: using EM_WORKAROUND_PYTHON_BUG_34780\n"); + CloseHandle(GetStdHandle(STD_INPUT_HANDLE)); + } + + STARTUPINFOW si; + PROCESS_INFORMATION pi; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + ZeroMemory(&pi, sizeof(pi)); + + dbg("pylauncher: running: %ls\n", command_line); + if (!CreateProcessW(NULL, command_line, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { + fprintf(stderr, "pylauncher: CreateProcess failed (%d): %ls\n", GetLastError(), command_line); + abort(); + } + WaitForSingleObject(pi.hProcess, INFINITE); + + DWORD exit_code; + GetExitCodeProcess(pi.hProcess, &exit_code); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + + dbg("pylauncher: done: %d\n", exit_code); + return exit_code; +} diff --git a/tools/pylauncher/pylauncher.exe b/tools/pylauncher/pylauncher.exe new file mode 100644 index 0000000000000..025391d077282 Binary files /dev/null and b/tools/pylauncher/pylauncher.exe differ