From 508b7cd003413c705989eb0d38fe17ffd8c56885 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 6 May 2026 21:42:44 +0100 Subject: [PATCH] fix: macos build missing shared libraries --- nix/edge-runtime.nix | 224 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 192 insertions(+), 32 deletions(-) diff --git a/nix/edge-runtime.nix b/nix/edge-runtime.nix index 86fb1442..78862a0d 100644 --- a/nix/edge-runtime.nix +++ b/nix/edge-runtime.nix @@ -27,18 +27,6 @@ let }; }; - # TODO: Missing "x86_64-darwin" supabase/rusty_v8 - #"x86_64-darwin" = { - # archive = { - # url = "https://github.com/supabase/rusty_v8/releases/download/v130.0.7/src_binding_release_x86_64-apple-darwin.rs"; - # sha256 = "sha256-VYWg+9WekcHBJWEq49eAAVpc6g/PPaoZDm/j2DKNQLY="; - # }; - # binding = { - # url = "https://github.com/supabase/rusty_v8/releases/download/v130.0.7/src_binding_release_x86_64-apple-darwin.rs"; - # sha256 = lib.fakeHash; - # }; - #}; - "aarch64-linux" = { archive = { url = "https://github.com/supabase/rusty_v8/releases/download/v130.0.7/librusty_v8_release_aarch64-unknown-linux-gnu.a.gz"; @@ -67,7 +55,7 @@ let build_step = rustPlatform.buildRustPackage (finalAttrs: { pname = "edge_runtime_build"; - version = "v1.73.3"; + version = "v1.73.15"; src = ../.; nativeBuildInputs = [ pkg-config curl cmake ]; buildInputs = [ openblas onnxruntime openssl zstd ]; @@ -90,6 +78,16 @@ let RUSTY_V8_ARCHIVE = v8Archive; RUSTY_V8_SRC_BINDING_PATH = v8Binding; DYLD_LIBRARY_PATH = "${onnxruntime}/lib"; + } // lib.optionalAttrs stdenv.isDarwin { + RUSTFLAGS = "-C debuginfo=0 -C link-arg=-Wl,-rpath,${onnxruntime}/lib"; + DYLD_FALLBACK_LIBRARY_PATH = "${onnxruntime}/lib"; + preBuild = '' + mkdir -p target/release target/release/deps + for lib in ${onnxruntime}/lib/libonnxruntime*.dylib*; do + ln -sf "$lib" "target/release/$(basename "$lib")" + ln -sf "$lib" "target/release/deps/$(basename "$lib")" + done + ''; }); in stdenv.mkDerivation { @@ -100,7 +98,8 @@ stdenv.mkDerivation { nativeBuildInputs = lib.optionals stdenv.isLinux [ patchelf ]; buildPhase = '' - mkdir -p $out/bin $out/lib + rootfs="$out" + mkdir -p "$rootfs/bin" "$rootfs/lib" binaries="edge-runtime" @@ -132,9 +131,9 @@ buildPhase = '' copy_dep() { local dep="$1" local libname=$(basename "$dep") - [ -f "$out/lib/$libname" ] && return # already copied + [ -f "$rootfs/lib/$libname" ] && return # already copied should_exclude "$libname" && return - [ -f "$dep" ] && cp "$dep" $out/lib/ 2>/dev/null || true + [ -f "$dep" ] && cp -L "$dep" "$rootfs/lib/$libname" 2>/dev/null || true } # Helper function to get the library file pattern based on platform @@ -148,40 +147,41 @@ buildPhase = '' # Copy binaries from cargo build to wrapped style for bin in $binaries; do - cp ${build_step}/bin/$bin $out/bin/.$bin-wrapped 2>/dev/null || true + cp ${build_step}/bin/$bin "$rootfs/bin/.$bin-wrapped" 2>/dev/null || true done # Seed: direct deps of binary + onnxruntime and all its siblings - for dep in $(get_deps $out/bin/.*-wrapped); do + for dep in $(get_deps "$rootfs"/bin/.*-wrapped); do copy_dep "$dep" done # Copy onnxruntime lib_pattern=$(get_lib_pattern) - cp ${onnxruntime}/lib/libonnxruntime$lib_pattern $out/lib/ 2>/dev/null || true + cp -P ${onnxruntime}/lib/libonnxruntime$lib_pattern "$rootfs/lib/" 2>/dev/null || true # Iterative crawl until no new deps appear for iteration in {1..5}; do - before_count=$(ls $out/lib/ | wc -l || echo "0") + before_count=$(ls "$rootfs/lib/" | wc -l || echo "0") - for lib in $out/lib/*; do + for lib in "$rootfs"/lib/*; do [ -f "$lib" ] || continue for dep in $(get_deps "$lib"); do copy_dep "$dep" done done - after=$(ls $out/lib/ | wc -l) + after=$(ls "$rootfs/lib/" | wc -l) echo "Iteration $iteration: $before_count -> $after libs" [ "$before_count" -eq "$after" ] && break done ''; installPhase = '' + rootfs="$out" # Create wrapper scripts and set up library paths for bin in $binaries; do - if [ -f $out/bin/.$bin-wrapped ]; then - cat > $out/bin/$bin << 'WRAPPER_EOF' + if [ -f "$rootfs/bin/.$bin-wrapped" ]; then + cat > "$rootfs/bin/$bin" << 'WRAPPER_EOF' #!/bin/sh SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" LIB_DIR="$SCRIPT_DIR/../lib" @@ -189,23 +189,26 @@ LIB_DIR="$SCRIPT_DIR/../lib" # For Linux, set LD_LIBRARY_PATH to include bundled libraries if [ "$(uname)" = "Linux" ]; then export LD_LIBRARY_PATH="$LIB_DIR''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + export ORT_DYLIB_PATH="''${ORT_DYLIB_PATH:-$LIB_DIR/libonnxruntime.so}" fi # For macOS, set DYLD_LIBRARY_PATH if [ "$(uname)" = "Darwin" ]; then export DYLD_LIBRARY_PATH="$LIB_DIR''${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + export ORT_DYLIB_PATH="''${ORT_DYLIB_PATH:-$LIB_DIR/libonnxruntime.dylib}" fi exec "$SCRIPT_DIR/.BINNAME-wrapped" "$@" WRAPPER_EOF - sed -i "s/BINNAME/$bin/g" $out/bin/$bin - chmod +x $out/bin/$bin + sed -i "s/BINNAME/$bin/g" "$rootfs/bin/$bin" + chmod +x "$rootfs/bin/$bin" fi done ''; postFixup = lib.optionalString stdenv.isLinux '' + rootfs="$out" # Determine the correct interpreter path based on architecture if [ "$(uname -m)" = "x86_64" ]; then INTERP="/lib64/ld-linux-x86-64.so.2" @@ -218,7 +221,7 @@ WRAPPER_EOF # On Linux, patch binaries to use system interpreter and relative library paths # This makes the bundle portable across Linux systems - for bin in $out/bin/.*-wrapped; do + for bin in "$rootfs"/bin/.*-wrapped; do if [ -f "$bin" ] && file "$bin" | grep -q ELF; then echo "Patching RPATH and interpreter for $bin" # Set interpreter to system dynamic linker for portability @@ -227,30 +230,84 @@ WRAPPER_EOF patchelf --set-rpath '$ORIGIN/../lib' "$bin" 2>/dev/null || true # Shrink RPATH to remove any unused paths patchelf --shrink-rpath "$bin" 2>/dev/null || true + strip --strip-unneeded "$bin" 2>/dev/null || true fi done # Patch shared libraries to use relative RPATH - for lib in $out/lib/*.so*; do + for lib in "$rootfs"/lib/*.so*; do if [ -f "$lib" ] && file "$lib" | grep -q ELF; then echo "Patching RPATH for $lib" # Set RPATH to $ORIGIN so libraries find other libraries in same directory patchelf --set-rpath '$ORIGIN' "$lib" 2>/dev/null || true # Shrink RPATH to remove any unused paths patchelf --shrink-rpath "$lib" 2>/dev/null || true + strip --strip-unneeded "$lib" 2>/dev/null || true fi done + + echo "Auditing Linux portable output" + unresolved="$( + for elf in "$rootfs"/bin/.*-wrapped "$rootfs"/lib/*.so*; do + [ -f "$elf" ] || continue + if file "$elf" | grep -q ELF; then + patchelf --print-rpath "$elf" 2>/dev/null | grep /nix/store | sed "s|^|$elf rpath -> |" || true + ldd "$elf" 2>/dev/null | awk -v file="$elf" -v rootfs="$rootfs" ' + /=> \// { path = $3 } + /^[[:space:]]*\// { path = $1 } + path !~ "^/nix/store/" { path = ""; next } + index(path, rootfs "/") == 1 { path = ""; next } + { + name = path + sub(/^.*\//, "", name) + if (name ~ /^(ld-linux.*|libc\.so.*|libc-.*\.so.*|libdl\.so.*|libpthread\.so.*|libm\.so.*|libresolv\.so.*|librt\.so.*)$/) { + path = "" + next + } + print file " -> " path + path = "" + } + ' + fi + done + )" + if [ -n "$unresolved" ]; then + echo "$unresolved" >&2 + exit 1 + fi '' + lib.optionalString stdenv.isDarwin '' + rootfs="$out" + chmod -R u+w "$rootfs" + + is_macho() { + file "$1" 2>/dev/null | grep -q "Mach-O" + } + + macho_files() { + find "$rootfs/bin" "$rootfs/lib" -type f 2>/dev/null | while read file_path; do + if is_macho "$file_path"; then + echo "$file_path" + fi + done + } + + read_rpaths() { + otool -l "$1" 2>/dev/null | awk ' + $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } + in_rpath && $1 == "path" { print $2; in_rpath = 0 } + ' + } + # On macOS, patch binaries to use relative library paths # This makes the bundle portable across macOS systems - for bin in $out/bin/.*-wrapped; do + for bin in "$rootfs"/bin/.*-wrapped; do if [ -f "$bin" ] && file "$bin" | grep -q "Mach-O"; then # Get all dylib dependencies from Nix store otool -L "$bin" | grep /nix/store | awk '{print $1}' | while read dep; do libname=$(basename "$dep") # Check if we have this library in our lib directory - if [ -f "$out/lib/$libname" ]; then + if [ -f "$rootfs/lib/$libname" ]; then echo "Patching $bin: $dep -> @rpath/$libname" install_name_tool -change "$dep" "@rpath/$libname" "$bin" 2>/dev/null || true fi @@ -261,7 +318,7 @@ WRAPPER_EOF done # Patch dylibs to use @rpath for their dependencies - for lib in $out/lib/*.dylib*; do + for lib in "$rootfs"/lib/*.dylib*; do if [ -f "$lib" ] && file "$lib" | grep -q "Mach-O"; then # First, fix the library's own ID to use @rpath libname=$(basename "$lib") @@ -273,13 +330,116 @@ WRAPPER_EOF # Then fix references to other libraries otool -L "$lib" | grep /nix/store | awk '{print $1}' | while read dep; do deplibname=$(basename "$dep") - if [ -f "$out/lib/$deplibname" ]; then + if [ -f "$rootfs/lib/$deplibname" ]; then echo "Patching $lib: $dep -> @rpath/$deplibname" install_name_tool -change "$dep" "@rpath/$deplibname" "$lib" 2>/dev/null || true fi done fi done + + echo "Completing Darwin dylib closure" + for iteration in 1 2 3 4 5; do + copied=0 + while read macho; do + rpaths="$(read_rpaths "$macho")" + + otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^@rpath/" { print $1 }' | while read dep; do + dep_name="$(basename "$dep")" + [ -e "$rootfs/lib/$dep_name" ] && continue + + candidate="" + while read rpath; do + [ -n "$rpath" ] || continue + case "$rpath" in + @loader_path) maybe="$(dirname "$macho")/$dep_name" ;; + @executable_path/../lib) maybe="$rootfs/lib/$dep_name" ;; + /nix/store/*) maybe="$rpath/$dep_name" ;; + *) maybe="" ;; + esac + if [ -n "$maybe" ] && [ -e "$maybe" ]; then + candidate="$maybe" + break + fi + done </dev/null || true)" + fi + + if [ -n "$candidate" ] && [ -e "$candidate" ]; then + cp -L "$candidate" "$rootfs/lib/$dep_name" + chmod u+w "$rootfs/lib/$dep_name" 2>/dev/null || true + touch "$rootfs/.darwin-deps-copied" + fi + done + + otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' | while read dep; do + dep_name="$(basename "$dep")" + [ -e "$rootfs/lib/$dep_name" ] && continue + if [ -e "$dep" ]; then + cp -L "$dep" "$rootfs/lib/$dep_name" + chmod u+w "$rootfs/lib/$dep_name" 2>/dev/null || true + touch "$rootfs/.darwin-deps-copied" + fi + done + done </dev/null || true + ;; + esac + + otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' | while read dep; do + dep_name="$(basename "$dep")" + if [ -e "$rootfs/lib/$dep_name" ]; then + install_name_tool -change "$dep" "@rpath/$dep_name" "$macho" 2>/dev/null || true + fi + done + + read_rpaths "$macho" | while read rpath; do + case "$rpath" in + /nix/store/*) install_name_tool -delete_rpath "$rpath" "$macho" 2>/dev/null || true ;; + esac + done + + strip -x "$macho" 2>/dev/null || true + codesign --force --sign - "$macho" 2>/dev/null || true + done </dev/null | awk -v file="$macho" 'NR > 1 && $1 ~ "^/nix/store/" { print file " -> " $1 }' + otool -l "$macho" 2>/dev/null | awk -v file="$macho" ' + $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } + in_rpath && $1 == "path" && $2 ~ "^/nix/store/" { print file " rpath -> " $2; in_rpath = 0 } + in_rpath && $1 == "path" { in_rpath = 0 } + ' + done <&2 + exit 1 + fi ''; meta = {