Add runtime detection for macFUSE 4.x/5.x rename protocol

build.rs

@@ -16,11 +16,13 @@ fn main() {
     } else if target_os == "macos" {
         pkg_config::Config::new()
             .atleast_version("2.6.0")
-            .probe("fuse") // for macFUSE 4.x
+            .probe("fuse") // for macFUSE
             .map_err(|e| eprintln!("{e}"))
             .unwrap();
         println!("cargo::rustc-cfg=fuser_mount_impl=\"libfuse2\"");
-        println!("cargo::rustc-cfg=feature=\"macfuse-4-compat\"");
+        // Note: We use runtime detection for macFUSE 4.x vs 5.x protocol differences
+        // in request.rs instead of the compile-time macfuse-4-compat feature.
+        // This allows the binary to work with both macFUSE 4.x and 5.x installations.
     } else {
         // First try to link with libfuse3
         if pkg_config::Config::new()

src/lib.rs

@@ -11,7 +11,7 @@
     unreachable_pub
 )]
 
-use libc::{ENOSYS, EPERM, c_int};
+use libc::{c_int, ENOSYS, EPERM};
 use log::warn;
 use mnt::mount_options::parse_options_from_args;
 #[cfg(feature = "serializable")]
@@ -24,9 +24,9 @@ use std::time::Duration;
 use std::time::SystemTime;
 use std::{convert::AsRef, io::ErrorKind};
 
-pub use crate::ll::fuse_abi::FUSE_ROOT_ID;
 use crate::ll::fuse_abi::consts::*;
-pub use crate::ll::{TimeOrNow, fuse_abi::consts};
+pub use crate::ll::fuse_abi::FUSE_ROOT_ID;
+pub use crate::ll::{fuse_abi::consts, TimeOrNow};
 use crate::mnt::mount_options::check_option_conflicts;
 use crate::session::MAX_WRITE_SIZE;
 pub use ll::fuse_abi::fuse_forget_one;
@@ -67,10 +67,25 @@ mod session;
 const INIT_FLAGS: u64 = FUSE_ASYNC_READ | FUSE_BIG_WRITES;
 // TODO: Add FUSE_EXPORT_SUPPORT
 
-/// On macOS, we additionally support case insensitiveness, volume renames and xtimes
+/// On macOS, we additionally support case insensitiveness, volume renames, xtimes,
+/// and extended rename operations (swap, exclusive).
+///
+/// We request FUSE_RENAME_SWAP and FUSE_RENAME_EXCL to enable extended rename support.
+/// When granted, macFUSE 5.x sends the extended 16-byte fuse_rename_in format with flags.
+/// Note: macFUSE 4.x had a bug where it sent extended format unconditionally, regardless
+/// of whether capabilities were granted. Our request.rs handles both formats via runtime
+/// detection for backward compatibility.
+///
+/// See: https://github.com/osxfuse/osxfuse/issues/839
+///
 /// TODO: we should eventually let the filesystem implementation decide which flags to set
 #[cfg(target_os = "macos")]
-const INIT_FLAGS: u64 = FUSE_ASYNC_READ | FUSE_CASE_INSENSITIVE | FUSE_VOL_RENAME | FUSE_XTIMES;
+const INIT_FLAGS: u64 = FUSE_ASYNC_READ
+    | FUSE_CASE_INSENSITIVE
+    | FUSE_VOL_RENAME
+    | FUSE_XTIMES
+    | FUSE_RENAME_SWAP
+    | FUSE_RENAME_EXCL;
 // TODO: Add FUSE_EXPORT_SUPPORT and FUSE_BIG_WRITES (requires ABI 7.10)
 
 const fn default_init_flags(#[allow(unused_variables)] capabilities: u64) -> u64 {

src/ll/argument.rs

@@ -85,6 +85,28 @@ impl<'a> ArgumentIterator<'a> {
         self.data = &rest[1..];
         Some(OsStr::from_bytes(out))
     }
+
+    /// Skip a fixed number of bytes. Returns `true` if successful, `false` if not enough data.
+    ///
+    /// This is useful for runtime protocol detection where we need to skip optional
+    /// fields based on the actual kernel protocol version.
+    pub(crate) fn skip_bytes(&mut self, count: usize) -> bool {
+        if self.data.len() >= count {
+            self.data = &self.data[count..];
+            true
+        } else {
+            false
+        }
+    }
+
+    /// Peek at the byte at the given offset without consuming it.
+    /// Returns `None` if the offset is beyond the remaining data.
+    ///
+    /// This is useful for runtime detection of protocol variants by inspecting
+    /// upcoming bytes before deciding how to parse.
+    pub(crate) fn peek_byte(&self, offset: usize) -> Option<u8> {
+        self.data.get(offset).copied()
+    }
 }
 
 #[cfg(test)]

src/ll/fuse_abi.rs

@@ -241,6 +241,12 @@ pub mod consts {
     #[cfg(feature = "abi-7-40")]
     pub const FUSE_PASSTHROUGH: u64 = 1 << 37; // filesystem wants to use passthrough files
 
+    // macOS-specific init flags (note: bits 25-26 overlap with Linux's FUSE_EXPLICIT_INVAL_DATA)
+    // See: https://github.com/osxfuse/fuse/blob/master/include/fuse_kernel.h
+    #[cfg(target_os = "macos")]
+    pub const FUSE_RENAME_SWAP: u64 = 1 << 25; // Enable atomic rename swap
+    #[cfg(target_os = "macos")]
+    pub const FUSE_RENAME_EXCL: u64 = 1 << 26; // Enable rename fail-if-exists
     #[cfg(target_os = "macos")]
     pub const FUSE_ALLOCATE: u64 = 1 << 27;
     #[cfg(target_os = "macos")]
@@ -541,14 +547,22 @@ pub(crate) struct fuse_mkdir_in {
     pub(crate) umask: u32,
 }
 
+/// Rename request structure (8 bytes).
+///
+/// On macOS with macFUSE, we request FUSE_RENAME_SWAP and FUSE_RENAME_EXCL
+/// capabilities during init. If granted, the kernel sends an extended 16-byte
+/// format (this struct plus flags: u32 + padding: u32). We use runtime detection
+/// to handle both formats, allowing compatibility across macFUSE versions.
+///
+/// Linux has FUSE_RENAME2 as a separate opcode for extended renames.
+///
+/// See:
+/// - Header: https://github.com/osxfuse/fuse/blob/master/include/fuse_kernel.h
+/// - Issue: https://github.com/osxfuse/osxfuse/issues/839
 #[repr(C)]
 #[derive(Debug, FromBytes, KnownLayout, Immutable)]
 pub(crate) struct fuse_rename_in {
     pub(crate) newdir: u64,
-    #[cfg(feature = "macfuse-4-compat")]
-    pub(crate) flags: u32,
-    #[cfg(feature = "macfuse-4-compat")]
-    pub(crate) padding: u32,
 }
 
 #[repr(C)]

src/ll/notify.rs

@@ -3,7 +3,7 @@ use std::{convert::TryInto, io::IoSlice, num::TryFromIntError};
 #[allow(unused)]
 use std::{ffi::OsStr, os::unix::ffi::OsStrExt};
 
-use smallvec::{SmallVec, smallvec};
+use smallvec::{smallvec, SmallVec};
 use zerocopy::{Immutable, IntoBytes};
 
 use super::fuse_abi as abi;

src/ll/reply.rs

@@ -9,9 +9,9 @@ use std::{
 use crate::FileType;
 
 use super::fuse_abi::FopenFlags;
-use super::{Errno, FileHandle, Generation, INodeNo, fuse_abi as abi};
+use super::{fuse_abi as abi, Errno, FileHandle, Generation, INodeNo};
 use super::{Lock, RequestId};
-use smallvec::{SmallVec, smallvec};
+use smallvec::{smallvec, SmallVec};
 use zerocopy::{Immutable, IntoBytes};
 
 const INLINE_DATA_THRESHOLD: usize = size_of::<u64>() * 4;

src/ll/request.rs

@@ -3,9 +3,9 @@
 //! A request represents information about a filesystem operation the kernel driver wants us to
 //! perform.
 
-use super::fuse_abi::{InvalidOpcodeError, fuse_in_header, fuse_opcode};
+use super::fuse_abi::{fuse_in_header, fuse_opcode, InvalidOpcodeError};
 
-use super::{Errno, Response, fuse_abi as abi};
+use super::{fuse_abi as abi, Errno, Response};
 #[cfg(feature = "serializable")]
 use serde::{Deserialize, Serialize};
 use std::{convert::TryFrom, fmt::Display, path::Path};
@@ -260,11 +260,11 @@ mod op {
     use crate::ll::Response;
 
     use super::{
-        super::{TimeOrNow, argument::ArgumentIterator},
+        super::{argument::ArgumentIterator, TimeOrNow},
         FilenameInDir, Request,
     };
     use super::{
-        FileHandle, INodeNo, Lock, LockOwner, Operation, RequestId, abi::consts::*, abi::*,
+        abi::consts::*, abi::*, FileHandle, INodeNo, Lock, LockOwner, Operation, RequestId,
     };
     use std::{
         convert::TryInto,
@@ -587,6 +587,9 @@ mod op {
         arg: &'a fuse_rename_in,
         name: &'a Path,
         newname: &'a Path,
+        /// Rename flags (macOS only, from extended format).
+        /// On other platforms or when macFUSE sends short format, this is 0.
+        flags: u32,
     }
     impl_request!(Rename<'_>);
     impl<'a> Rename<'a> {
@@ -602,6 +605,14 @@ mod op {
                 name: self.newname,
             }
         }
+        /// Rename flags (macOS only: RENAME_SWAP, RENAME_EXCL).
+        ///
+        /// On macOS, if FUSE_RENAME_SWAP or FUSE_RENAME_EXCL capabilities were granted
+        /// during init, the kernel sends extended rename requests with flags.
+        /// On other platforms or when macFUSE sends the short format, this returns 0.
+        pub(crate) fn flags(&self) -> u32 {
+            self.flags
+        }
     }
 
     /// Create a hard link.
@@ -1651,12 +1662,74 @@ mod op {
                 header,
                 name: data.fetch_str()?.as_ref(),
             }),
-            fuse_opcode::FUSE_RENAME => Operation::Rename(Rename {
-                header,
-                arg: data.fetch()?,
-                name: data.fetch_str()?.as_ref(),
-                newname: data.fetch_str()?.as_ref(),
-            }),
+            fuse_opcode::FUSE_RENAME => {
+                let arg: &fuse_rename_in = data.fetch()?;
+
+                // macOS macFUSE protocol version detection for FUSE_RENAME
+                // =========================================================
+                //
+                // Background:
+                // macFUSE supports extended rename operations (RENAME_SWAP, RENAME_EXCL)
+                // via flags in the rename request. When these capabilities are negotiated
+                // during FUSE_INIT, macFUSE sends an extended 16-byte fuse_rename_in:
+                //   - newdir: u64 (8 bytes)
+                //   - flags: u32 (4 bytes)
+                //   - padding: u32 (4 bytes)
+                //
+                // The problem:
+                // - macFUSE 4.x had a bug where it sent the extended format unconditionally,
+                //   regardless of whether capabilities were actually granted.
+                // - macFUSE 5.x fixed this: it only sends extended format when capabilities
+                //   ARE granted; otherwise it sends the short 8-byte format.
+                //
+                // Our solution:
+                // We request FUSE_RENAME_SWAP/FUSE_RENAME_EXCL at INIT (see lib.rs), which
+                // should cause macFUSE 5.x to grant them and send extended format. However,
+                // we use runtime detection as a safety measure because:
+                //   1. The kernel may refuse to grant the capabilities
+                //   2. We need backward compatibility with macFUSE 4.x behavior
+                //
+                // Detection heuristic:
+                // Filenames cannot start with null bytes or ASCII control characters (< 32).
+                // If the first byte after newdir is in the control range, we have extended
+                // format (flags field starts there). If it's >= 32, it's the start of a
+                // filename, indicating short format.
+                //
+                // References:
+                // - Issue: https://github.com/osxfuse/osxfuse/issues/839
+                // - macFUSE kernel header: https://github.com/osxfuse/fuse/blob/master/include/fuse_kernel.h
+                #[cfg(target_os = "macos")]
+                let flags = {
+                    let first_byte = data.peek_byte(0);
+                    if first_byte.is_some_and(|b| b < 32) {
+                        // Extended format detected: extract flags as little-endian u32
+                        let flags_bytes = [
+                            data.peek_byte(0).unwrap_or(0),
+                            data.peek_byte(1).unwrap_or(0),
+                            data.peek_byte(2).unwrap_or(0),
+                            data.peek_byte(3).unwrap_or(0),
+                        ];
+                        let flags = u32::from_le_bytes(flags_bytes);
+                        // Skip flags (4 bytes) + padding (4 bytes) = 8 bytes total
+                        data.skip_bytes(8);
+                        flags
+                    } else {
+                        // Short format: filename starts immediately, no flags available
+                        0
+                    }
+                };
+
+                #[cfg(not(target_os = "macos"))]
+                let flags = 0u32;
+
+                Operation::Rename(Rename {
+                    header,
+                    arg,
+                    name: data.fetch_str()?.as_ref(),
+                    newname: data.fetch_str()?.as_ref(),
+                    flags,
+                })
+            }
             fuse_opcode::FUSE_LINK => Operation::Link(Link {
                 header,
                 arg: data.fetch()?,

src/mnt/fuse2.rs

@@ -1,4 +1,4 @@
-use super::{MountOption, fuse2_sys::*, with_fuse_args};
+use super::{fuse2_sys::*, with_fuse_args, MountOption};
 use log::warn;
 use std::{
     ffi::CString,

src/mnt/fuse3.rs

@@ -2,11 +2,11 @@ use super::fuse3_sys::{
     fuse_lowlevel_ops, fuse_session_destroy, fuse_session_fd, fuse_session_mount, fuse_session_new,
     fuse_session_unmount,
 };
-use super::{MountOption, with_fuse_args};
+use super::{with_fuse_args, MountOption};
 use log::warn;
 use std::os::fd::BorrowedFd;
 use std::{
-    ffi::{CString, c_void},
+    ffi::{c_void, CString},
     fs::File,
     io,
     os::unix::{ffi::OsStrExt, io::FromRawFd},

src/mnt/fuse_pure.rs

@@ -7,10 +7,10 @@
 #![allow(missing_docs)]
 
 use super::is_mounted;
-use super::mount_options::{MountOption, option_to_string};
+use super::mount_options::{option_to_string, MountOption};
 use log::{debug, error};
-use nix::fcntl::{FcntlArg, FdFlag, OFlag, fcntl};
-use nix::sys::socket::{ControlMessageOwned, MsgFlags, SockaddrStorage, recvmsg};
+use nix::fcntl::{fcntl, FcntlArg, FdFlag, OFlag};
+use nix::sys::socket::{recvmsg, ControlMessageOwned, MsgFlags, SockaddrStorage};
 use std::ffi::{CStr, CString, OsStr};
 use std::fs::File;
 #[cfg(any(

src/mnt/mod.rs

@@ -46,12 +46,12 @@ fn with_fuse_args<T, F: FnOnce(&fuse_args) -> T>(options: &[MountOption], f: F)
     })
 }
 
-#[cfg(fuser_mount_impl = "pure-rust")]
-pub(crate) use fuse_pure::Mount;
 #[cfg(fuser_mount_impl = "libfuse2")]
 pub(crate) use fuse2::Mount;
 #[cfg(fuser_mount_impl = "libfuse3")]
 pub(crate) use fuse3::Mount;
+#[cfg(fuser_mount_impl = "pure-rust")]
+pub(crate) use fuse_pure::Mount;
 use std::ffi::CStr;
 
 #[inline]
@@ -84,7 +84,7 @@ fn libc_umount(mnt: &CStr) -> io::Result<()> {
 /// yet destroyed by the kernel.
 #[cfg(any(test, fuser_mount_impl = "pure-rust"))]
 fn is_mounted(fuse_device: &File) -> bool {
-    use nix::poll::{PollFd, PollFlags, PollTimeout, poll};
+    use nix::poll::{poll, PollFd, PollFlags, PollTimeout};
     use std::os::unix::io::AsFd;
     use std::slice;
 

src/reply.rs

@@ -8,12 +8,13 @@
 
 use crate::ll::fuse_abi::FopenFlags;
 use crate::ll::{
-    self, Generation,
+    self,
     reply::{DirEntPlusList, DirEntryPlus},
+    Generation,
 };
 use crate::ll::{
-    INodeNo,
     reply::{DirEntList, DirEntOffset, DirEntry},
+    INodeNo,
 };
 #[cfg(feature = "abi-7-40")]
 use crate::passthrough::BackingId;
@@ -701,7 +702,7 @@ mod test {
     use super::*;
     use crate::{FileAttr, FileType};
     use std::io::IoSlice;
-    use std::sync::mpsc::{SyncSender, sync_channel};
+    use std::sync::mpsc::{sync_channel, SyncSender};
     use std::thread;
     use std::time::{Duration, UNIX_EPOCH};
     use zerocopy::{Immutable, IntoBytes};