Refactor to use workers for all file access

fileglancer/apps/core.py

@@ -147,7 +147,7 @@ async def _ensure_repo_cache(url: str, pull: bool = False,
 
     When username is provided, the work is delegated to a worker subprocess
     that runs with the target user's real UID/GID, avoiding the process-wide
-    euid race condition that EffectiveUserContext has with concurrent async
+    euid race condition that seteuid/setegid has with concurrent async
     requests.  When username is None, git commands run in-process (used by
     the worker subprocess itself, or in single-user dev mode).
     """

fileglancer/filestore.py

@@ -667,6 +667,35 @@ def stream_file_range(self, path: str = None, start: int = 0, end: int = 0, buff
                 file_handle.close()
 
 
+    @staticmethod
+    def _stream_contents(file_handle, buffer_size: int = DEFAULT_BUFFER_SIZE) -> Generator[bytes, None, None]:
+        """Stream from an open file handle. Handle is closed when done."""
+        try:
+            while True:
+                chunk = file_handle.read(buffer_size)
+                if not chunk:
+                    break
+                yield chunk
+        finally:
+            file_handle.close()
+
+    @staticmethod
+    def _stream_range(start: int, end: int, content_length: int,
+                      file_handle, buffer_size: int = DEFAULT_BUFFER_SIZE) -> Generator[bytes, None, None]:
+        """Stream a byte range from an open file handle. Handle is closed when done."""
+        try:
+            file_handle.seek(start)
+            remaining = content_length
+            while remaining > 0:
+                chunk_size = min(buffer_size, remaining)
+                chunk = file_handle.read(chunk_size)
+                if not chunk:
+                    break
+                yield chunk
+                remaining -= len(chunk)
+        finally:
+            file_handle.close()
+
     def rename_file_or_dir(self, old_path: str, new_path: str):
         """
         Rename a file at the given old path to the new path.

fileglancer/settings.py

@@ -100,6 +100,10 @@ class Settings(BaseSettings):
     # Useful for setting up scheduler env (e.g., /misc/lsf/conf/profile.lsf).
     env_source_script: Optional[str] = None
 
+    # Worker pool settings
+    worker_pool_max_workers: int = 50
+    worker_pool_idle_timeout: int = 300  # seconds
+
     # Cluster / Apps settings (mirrors py-cluster-api ClusterConfig)
     cluster: ClusterSettings = ClusterSettings()