From 747296a8092c45ca73750cbfe58a570db078c3ca Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Sun, 1 Mar 2026 00:27:09 +0000 Subject: [PATCH] usdm: add hugepage allocation fallback for VFIO When hugepages are configured but none are free (e.g. all consumed by DPDK), QATlib VFIO hugepage allocations fail with no fallback, breaking all QAT hardware offload. Add a three-layer defense: 1. Init-time check: Read free_hugepages from sysfs during __qae_vfio_init_hugepages(). If zero, disable hugepage mode immediately. 2. Runtime fallback: If hugepage slab allocation fails and no hugepage slab has ever been successfully allocated, disable hugepage mode and fall back to regular (4KB) VFIO memory allocation via ioctl path. 3. Mixed-state guard: If hugepage slabs already exist, refuse to fall back to avoid mixing 2MB and 4KB page table entries which would corrupt IOVA mappings. Also fix store_mmap_range() to derive page table granularity from the actual slab type (p_ctrl_blk->type) rather than the global hugepage mode flag, ensuring correctness after fallback. Signed-off-by: Giovanni Cabiddu --- .../libusdm_drv/user_space/qae_mem_common.c | 2 +- .../user_space/qae_mem_hugepage_utils.h | 3 ++ .../vfio/qae_mem_hugepage_utils_vfio.c | 44 +++++++++++++++++++ .../user_space/vfio/qae_mem_utils_vfio.c | 4 ++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/quickassist/utilities/libusdm_drv/user_space/qae_mem_common.c b/quickassist/utilities/libusdm_drv/user_space/qae_mem_common.c index 6b1555fe..c26544a5 100644 --- a/quickassist/utilities/libusdm_drv/user_space/qae_mem_common.c +++ b/quickassist/utilities/libusdm_drv/user_space/qae_mem_common.c @@ -413,7 +413,7 @@ void *__qae_alloc_addr(size_t size, p_ctrl_blk->virt_addr, p_ctrl_blk->phy_addr, p_ctrl_blk->size, - __qae_hugepage_enabled()); + HUGE_PAGE == p_ctrl_blk->type); if (LARGE == mem_type) { diff --git a/quickassist/utilities/libusdm_drv/user_space/qae_mem_hugepage_utils.h b/quickassist/utilities/libusdm_drv/user_space/qae_mem_hugepage_utils.h index a7821ee4..2ff741f2 100644 --- a/quickassist/utilities/libusdm_drv/user_space/qae_mem_hugepage_utils.h +++ b/quickassist/utilities/libusdm_drv/user_space/qae_mem_hugepage_utils.h @@ -57,4 +57,7 @@ int __qae_vfio_init_hugepages(void); API_LOCAL int __qae_hugepage_enabled(void); + +API_LOCAL +int __qae_vfio_disable_hugepages(void); #endif diff --git a/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_hugepage_utils_vfio.c b/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_hugepage_utils_vfio.c index 548d6686..87a8bb92 100644 --- a/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_hugepage_utils_vfio.c +++ b/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_hugepage_utils_vfio.c @@ -41,6 +41,7 @@ #endif static bool g_hugepages_enabled = false; +static bool g_hugepages_allocated = false; static size_t g_num_hugepages = 0; static const char sys_dir_path[] = "/sys/kernel/mm/hugepages"; extern int vfio_container_fd; @@ -197,10 +198,30 @@ STATIC int mem_virt2phy(const void *virtaddr, uint64_t *physaddr_ptr) API_LOCAL int __qae_vfio_init_hugepages() { + char free_hp_path[HUGEPAGE_SYSFS_PATH_SIZE] = { }; + unsigned long free_hp = 0; int ret = 0; + if (get_num_hugepages_per_system(HUGEPAGE_SYS_NODE)) return -EIO; + if (g_num_hugepages > 0) + { + /* + * Check if any hugepages are actually free. + * If all hugepages are consumed (e.g. by DPDK), do not enable + * hugepage mode to avoid allocation failures at runtime. + */ + snprintf(free_hp_path, sizeof(free_hp_path), "%s/%s/free_hugepages", + sys_dir_path, HUGEPAGE_SYS_NODE); + if (parse_sysfs_value(free_hp_path, &free_hp) == 0 && free_hp == 0) + { + CMD_DEBUG("No free hugepages available (nr_hugepages=%zu). " + "Disabling hugepage mode.\n", g_num_hugepages); + g_num_hugepages = 0; + } + } + if (g_num_hugepages > 0) { g_hugepages_enabled = true; @@ -224,6 +245,27 @@ int __qae_hugepage_enabled() return g_hugepages_enabled; } +API_LOCAL +int __qae_vfio_disable_hugepages() +{ + if (g_hugepages_allocated) + { + CMD_ERROR("Hugepage allocation failed but hugepage slabs already " + "exist. Cannot fall back to regular memory.\n"); + return 0; + } + + CMD_DEBUG("Hugepage allocation failed, disabling hugepage mode. " + "Falling back to regular memory allocation.\n"); + + g_hugepages_enabled = false; + __qae_set_free_page_table_fptr(free_page_table); + __qae_set_loadaddr_fptr(load_addr); + __qae_set_loadkey_fptr(load_key); + + return 1; +} + STATIC void *__qae_vfio_hugepage_mmap_addr(const size_t size) { void *addr = NULL; @@ -350,6 +392,7 @@ dev_mem_info_t *__qae_vfio_hugepage_alloc_slab(const int fd, save_slab_to_tmp_list(slab); #endif /* This is required for adding into hash table.*/ + g_hugepages_allocated = true; return slab; } @@ -377,6 +420,7 @@ dev_mem_info_t *__qae_vfio_hugepage_alloc_slab(const int fd, slab->flag_pinned = PINNED; #endif + g_hugepages_allocated = true; return slab; error: diff --git a/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_utils_vfio.c b/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_utils_vfio.c index 21de89dd..496449d7 100644 --- a/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_utils_vfio.c +++ b/quickassist/utilities/libusdm_drv/user_space/vfio/qae_mem_utils_vfio.c @@ -548,6 +548,8 @@ dev_mem_info_t *__qae_alloc_slab(const int fd, if (HUGE_PAGE == type) { slab = __qae_vfio_hugepage_alloc_slab(fd, size, node, type, alignment); + if (!slab && __qae_vfio_disable_hugepages()) + slab = ioctl_alloc_slab(fd, size, alignment, node, SMALL); } else { @@ -580,6 +582,8 @@ dev_mem_info_t *__qae_alloc_slab(const int fd, if (HUGE_PAGE == type) { slab = __qae_vfio_hugepage_alloc_slab(fd, size, node, type, alignment); + if (!slab && __qae_vfio_disable_hugepages()) + slab = ioctl_alloc_slab(fd, size, alignment, node, SMALL); } else {