ggml: Load ggml library from detected paths

- Auto-select lib/ or bin/ directories
- Add backend loading functions

Author: JamePeng

CMakeLists.txt

@@ -17,23 +17,31 @@ function(llama_cpp_python_install_target target)
         return()
     endif()
 
-    install(
-        TARGETS ${target}
-        LIBRARY DESTINATION llama_cpp/lib
-        RUNTIME DESTINATION llama_cpp/lib
-        ARCHIVE DESTINATION llama_cpp/lib
-        FRAMEWORK DESTINATION llama_cpp/lib
-        RESOURCE DESTINATION llama_cpp/lib
+    # Define install destinations to avoid code duplication
+    set(INSTALL_DIRS
+        "${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib"
+        "${SKBUILD_PLATLIB_DIR}/llama_cpp/lib"
     )
 
-    # Windows DLL dependencies
-    if (WIN32)
+    foreach(DIR ${INSTALL_DIRS})
         install(
-            FILES $<TARGET_RUNTIME_DLLS:${target}>
-            DESTINATION llama_cpp/lib
-            OPTIONAL
+            TARGETS ${target}
+            LIBRARY DESTINATION ${DIR}
+            RUNTIME DESTINATION ${DIR}
+            ARCHIVE DESTINATION ${DIR}
+            FRAMEWORK DESTINATION ${DIR}
+            RESOURCE DESTINATION ${DIR}
         )
-    endif()
+
+        # Automatically handle Windows DLL installation for each target
+        if (WIN32)
+            install(
+                FILES $<TARGET_RUNTIME_DLLS:${target}>
+                DESTINATION ${DIR}
+                OPTIONAL # Prevent errors if the target has no DLLs
+            )
+        endif()
+    endforeach()
 
     # Proper RPATH handling
     if(UNIX)

llama_cpp/_ctypes_extensions.py

@@ -20,25 +20,23 @@
 
 
 # Load the library
-def load_shared_library(lib_base_name: str, base_path: pathlib.Path):
-    """Platform independent shared library loader"""
-    # Searching for the library in the current directory under the name "libllama" (default name
-    # for llamacpp) and "llama" (default name for this repo)
-    lib_paths: List[pathlib.Path] = []
-    # Determine the file extension based on the platform
+def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list[pathlib.Path]]):
+    if isinstance(base_paths, pathlib.Path):
+        base_paths = [base_paths]
+
+    lib_names = []
+
     if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"):
-        lib_paths += [
-            base_path / f"lib{lib_base_name}.so",
-        ]
+        lib_names = [f"lib{lib_base_name}.so"]
     elif sys.platform == "darwin":
-        lib_paths += [
-            base_path / f"lib{lib_base_name}.so",
-            base_path / f"lib{lib_base_name}.dylib",
+        lib_names = [
+            f"lib{lib_base_name}.dylib",
+            f"lib{lib_base_name}.so",
         ]
     elif sys.platform == "win32":
-        lib_paths += [
-            base_path / f"{lib_base_name}.dll",
-            base_path / f"lib{lib_base_name}.dll",
+        lib_names = [
+            f"{lib_base_name}.dll",
+            f"lib{lib_base_name}.dll",
         ]
     else:
         raise RuntimeError("Unsupported platform")
@@ -47,11 +45,13 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path):
 
     # Add the library directory to the DLL search path on Windows (if needed)
     if sys.platform == "win32":
-        os.add_dll_directory(str(base_path))
+        for base_path in base_paths:
+            os.add_dll_directory(str(base_path))
         os.environ["PATH"] = str(base_path) + os.pathsep + os.environ["PATH"]
 
-    if sys.platform == "win32" and sys.version_info >= (3, 8):
-        os.add_dll_directory(str(base_path))
+    if sys.platform == "win32" and sys.version_info >= (3, 9):
+        for base_path in base_paths:
+            os.add_dll_directory(str(base_path))
         if "CUDA_PATH" in os.environ:
             cuda_path = os.environ["CUDA_PATH"]
             sub_dirs_to_add = [
@@ -75,16 +75,21 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path):
 
         cdll_args["winmode"] = ctypes.RTLD_GLOBAL
 
+    errors = []
+
     # Try to load the shared library, handling potential errors
-    for lib_path in lib_paths:
-        if lib_path.exists():
-            try:
-                return ctypes.CDLL(str(lib_path), **cdll_args)  # type: ignore
-            except Exception as e:
-                raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}")
-
-    raise FileNotFoundError(
-        f"Shared library with base name '{lib_base_name}' not found"
+    for base_path in base_paths:
+        for lib_name in lib_names:
+            lib_path = base_path / lib_name
+            if lib_path.exists():
+                try:
+                    return ctypes.CDLL(str(lib_path), **cdll_args)
+                except Exception as e:
+                    errors.append(f"{lib_path}: {e}")
+
+    raise RuntimeError(
+        f"Failed to load '{lib_base_name}' from {base_paths}\n"
+        + "\n".join(errors)
     )
 
 

llama_cpp/_ggml.py

@@ -7,7 +7,11 @@
 import pathlib
 import ctypes
 
-import llama_cpp._ctypes_extensions as ctypes_ext
+from llama_cpp._ctypes_extensions import (
+    load_shared_library,
+    byref,
+    ctypes_function_for_shared_library,
+)
 
 from typing import (
     Callable,
@@ -17,8 +21,15 @@
     TYPE_CHECKING,
 )
 
-libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib"
-libggml = ctypes_ext.load_shared_library("ggml", libggml_base_path)
+libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
+libggml_base_paths = [
+    libggml_base_path / "lib",
+    libggml_base_path / "bin",
+]
+
+libggml = load_shared_library("ggml", libggml_base_paths)
+
+ggml_function = ctypes_function_for_shared_library(libggml)
 
 # // ====== ggml.h ======
 
@@ -361,3 +372,20 @@ class ggml_opt_optimizer_params(ctypes.Structure):
 ggml_backend_sched_eval_callback = ctypes.CFUNCTYPE(
     ctypes.c_bool, ctypes.c_void_p, ctypes.c_bool, ctypes.c_void_p
 )
+
+# //
+# // Backend registry
+# //
+
+# // Load all known backends from dynamic libraries
+# GGML_API void               ggml_backend_load_all(void);
+@ggml_function("ggml_backend_load_all", [], None)
+def ggml_backend_load_all():
+    """Load all known backends from dynamic libraries"""
+    ...
+
+# GGML_API void               ggml_backend_load_all_from_path(const char * dir_path);
+@ggml_function("ggml_backend_load_all_from_path", [ctypes.c_char_p], None)
+def ggml_backend_load_all_from_path(dir_path: ctypes.c_char_p):
+    """Load all known backends from path"""
+    ...