From 0021378d6b302ee5421be048cfd2219c8aef730a Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Fri, 21 Nov 2025 15:59:57 +0800 Subject: [PATCH 1/7] gen-kdump-sysconfig.sh: Fix shfmt check Signed-off-by: Pingfan Liu --- gen-kdump-sysconfig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen-kdump-sysconfig.sh b/gen-kdump-sysconfig.sh index a08d7e1a..c26bcbdc 100755 --- a/gen-kdump-sysconfig.sh +++ b/gen-kdump-sysconfig.sh @@ -49,7 +49,7 @@ fi # # Generate the config file # -cat < Date: Fri, 12 Sep 2025 11:40:41 +0800 Subject: [PATCH 2/7] kdump-lib: Extend _crashkernel_add() with a param Currently, crashkernel_add() adjusts each range in the 'crashkernel=' command line by adding a delta value. This adjustment is necessary because certain hardware features require significant additional memory (up to 1GB on some ARM64 machines). While this approach works well for traditional installations, OSTree deployments present a challenge: kdump cannot detect the hardware environment on the target platform. One solution is to maximize the memory reserved for the kdump kernel. However, while this increased allocation ensures kdump functionality, it raises the risk of OOM conditions in the production kernel, particularly on small-memory machines that typically lack such hardware features (Mellanox network cards, SMMU, SME). This patch extends crashkernel_add() with a parameter to skip the first N ranges in the 'crashkernel=' command line. Later, we can use it to skip the first range. Signed-off-by: Pingfan Liu --- kdump-lib.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kdump-lib.sh b/kdump-lib.sh index 4196b0e5..4af8329f 100755 --- a/kdump-lib.sh +++ b/kdump-lib.sh @@ -946,13 +946,16 @@ _crashkernel_parse() # $1 crashkernel command line parameter # $2 size to be added +# $3 optionally skip the first n items in command line _crashkernel_add() { - local ck delta ret + local ck delta skip ret local range size offset + local count=0 ck="$1" delta="$2" + skip="${3:-0}" # Default to 0 if third parameter not provided ret="" while IFS=';' read -r size range offset; do @@ -967,8 +970,12 @@ _crashkernel_add() ret+="$range:" fi - size=$(memsize_add "$size" "$delta") || return 1 + if ((count >= skip)); then + size=$(memsize_add "$size" "$delta") || return 1 + fi + ret+="$size," + ((count++)) done < <(_crashkernel_parse "$ck") echo "${ret%,}" From b78a90980595f14561744213866e724df8c52159 Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Mon, 29 Sep 2025 19:24:13 +0800 Subject: [PATCH 3/7] kdump-lib_spec: Extend test for _crashkernel_add() Signed-off-by: Pingfan Liu --- spec/kdump-lib_spec.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/spec/kdump-lib_spec.sh b/spec/kdump-lib_spec.sh index 8ced9172..77b7bdaa 100644 --- a/spec/kdump-lib_spec.sh +++ b/spec/kdump-lib_spec.sh @@ -51,7 +51,7 @@ Describe 'kdump-lib' End Describe "_crashkernel_add()" - Context "For valid input values" + Context "For valid input values with two arguments (skip defaults to 0)" Parameters "2G-4G:256M,4G-64G:320M,64G-:576M" "100M" "2G-4G:356M,4G-64G:420M,64G-:676M" "2G-4G:256M" "100" "2G-4G:268435556" # avoids any rounding when size % 1024 != 0 @@ -74,6 +74,25 @@ Describe 'kdump-lib' The output should equal "$3" End End + + Context "For valid input values with three arguments (explicit skip)" + Parameters + "2G-4G:256M,4G-64G:320M,64G-:576M" "100M" "1" "2G-4G:256M,4G-64G:420M,64G-:676M" + "2G-4G:256M,4G-64G:320M,64G-:576M" "100M" "2" "2G-4G:256M,4G-64G:320M,64G-:676M" + "2G-4G:256M,4G-64G:320M,64G-:576M" "100M" "3" "2G-4G:256M,4G-64G:320M,64G-:576M" + "2G-4G:256M,4G-64G:320M,64G-:576M@4G" "100M" "1" "2G-4G:256M,4G-64G:420M,64G-:676M@4G" + "2G-4G:1G,4G-64G:2G,64G-:3G@4G" "100M" "1" "2G-4G:1G,4G-64G:2148M,64G-:3172M@4G" + "2G-4G:10000K,4G-64G:20000K" "100M" "1" "2G-4G:10000K,4G-64G:122400K" + "1M@1G" "1K" "1" "1M@1G" + "128G-1T:4G,10T-100T:1T" "1G" "1" "128G-1T:4G,10T-100T:1025G" + "1K,low" "1" "1" "1K,low" + End + It "should add delta to values after ':' starting after skip count" + When call _crashkernel_add "$1" "$2" "$3" + The output should equal "$4" + End + End + Context "For invalid input values" Parameters "2G-4G:256M.4G-64G:320M" "100M" From 015e0f935bb6512c0a0980a5fd0b90c313891512 Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Mon, 22 Sep 2025 11:14:17 +0800 Subject: [PATCH 4/7] kdump-lib: Skip adding reserved memory for small-memory machine This increased "crashkernel=" raises the risk of OOM conditions in the production kernel, particularly on small-memory machines that typically lack such hardware features (Mellanox network cards, SMMU). So we skip that case on aarch64. For other arches, the above factors are not considered, so ignore them for the time being. Signed-off-by: Pingfan Liu --- kdump-lib.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kdump-lib.sh b/kdump-lib.sh index 4af8329f..9c45b7d6 100755 --- a/kdump-lib.sh +++ b/kdump-lib.sh @@ -989,6 +989,7 @@ kdump_get_arch_recommend_crashkernel() { local _arch _ck_cmdline _dump_mode local _delta=0 + local _skip=0 if [[ -z $1 ]]; then if is_fadump_capable; then @@ -1015,6 +1016,8 @@ kdump_get_arch_recommend_crashkernel() else _running_kernel=$2 fi + # skip adding additional memory for small-memory machine + _skip=1 # the naming convention of 64k variant suffixes with +64k, e.g. "vmlinuz-5.14.0-312.el9.aarch64+64k" if echo "$_running_kernel" | grep -q 64k; then @@ -1038,7 +1041,7 @@ kdump_get_arch_recommend_crashkernel() fi fi - echo -n "$(_crashkernel_add "$_ck_cmdline" "${_delta}M")" + echo -n "$(_crashkernel_add "$_ck_cmdline" "${_delta}M" "$_skip")" } # return recommended size based on current system RAM size From 1ef61a64117f8338a11637403a3d749b06426340 Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Thu, 4 Sep 2025 10:34:43 +0800 Subject: [PATCH 5/7] kdumpctl: Move _is_osbuild() to kdump-lib.sh _is_osbuild() will be used in kdump-lib.sh, so moving it from kdumpctl to kdump-lib.sh Signed-off-by: Pingfan Liu --- kdump-lib.sh | 19 +++++++++++++++++++ kdumpctl | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/kdump-lib.sh b/kdump-lib.sh index 9c45b7d6..9e42e012 100755 --- a/kdump-lib.sh +++ b/kdump-lib.sh @@ -38,6 +38,25 @@ is_sme_or_sev_active() journalctl -q --dmesg --grep "^Memory Encryption Features active: AMD (SME|SEV)$" > /dev/null 2>&1 } +# read the value of an environ variable from given environ file path +# +# The environment variable entries in /proc/[pid]/environ are separated +# by null bytes instead of by spaces. +# +# $1: environment variable +# $2: environ file path +read_proc_environ_var() +{ + local _var=$1 _environ_path=$2 + sed -n -E "s/.*(^|\x00)${_var}=([^\x00]*).*/\2/p" < "$_environ_path" +} + +_OSBUILD_ENVIRON_PATH='/proc/1/environ' +_is_osbuild() +{ + [[ $(read_proc_environ_var container "$_OSBUILD_ENVIRON_PATH") == bwrap-osbuild ]] +} + has_command() { [[ -x $(command -v "$1") ]] diff --git a/kdumpctl b/kdumpctl index cb10f5bd..6d5b3b8b 100755 --- a/kdumpctl +++ b/kdumpctl @@ -1970,25 +1970,6 @@ reset_crashkernel_after_update() done } -# read the value of an environ variable from given environ file path -# -# The environment variable entries in /proc/[pid]/environ are separated -# by null bytes instead of by spaces. -# -# $1: environment variable -# $2: environ file path -read_proc_environ_var() -{ - local _var=$1 _environ_path=$2 - sed -n -E "s/.*(^|\x00)${_var}=([^\x00]*).*/\2/p" < "$_environ_path" -} - -_OSBUILD_ENVIRON_PATH='/proc/1/environ' -_is_osbuild() -{ - [[ $(read_proc_environ_var container "$_OSBUILD_ENVIRON_PATH") == bwrap-osbuild ]] -} - reset_crashkernel_for_installed_kernel() { local _installed_kernel _grub_entry_index _kernel _old_ck _old_fadump From 532ad212b58e191aba3c4d824abff1379433108f Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Mon, 23 Jun 2025 09:58:45 +0800 Subject: [PATCH 6/7] crashkernel: Maximize the reserved size in osbuild case At present, the deduction of the proper crashkernel value depends on detecting platform details. However, more and more platforms are now deployed using OSTree images, which means kdump-utils cannot retrieve complete platform information. This often leads to an underestimation of the memory required by the kdump kernel, increasing the risk of out-of-memory (OOM) issues. To mitigate this situation, we choose to maximize the reserved crashkernel size in the osbuild case, and recommend users to update the kernel command line and release the excess reserved memory after reboot (addressed in the next patch). Signed-off-by: Pingfan Liu --- kdump-lib.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/kdump-lib.sh b/kdump-lib.sh index 9e42e012..1dd83420 100755 --- a/kdump-lib.sh +++ b/kdump-lib.sh @@ -16,6 +16,7 @@ FADUMP_REGISTER_SYS_NODE="/sys/kernel/fadump/registered" FADUMP_APPEND_ARGS_SYS_NODE="/sys/kernel/fadump/bootargs_append" # shellcheck disable=SC2034 FENCE_KDUMP_CONFIG_FILE="/etc/sysconfig/fence_kdump" +maximize_crashkernel=false is_fadump_capable() { @@ -35,7 +36,7 @@ is_aws_aarch64() is_sme_or_sev_active() { - journalctl -q --dmesg --grep "^Memory Encryption Features active: AMD (SME|SEV)$" > /dev/null 2>&1 + $maximize_crashkernel || journalctl -q --dmesg --grep "^Memory Encryption Features active: AMD (SME|SEV)$" > /dev/null 2>&1 } # read the value of an environ variable from given environ file path @@ -850,12 +851,19 @@ get_recommend_size() has_mlx5() { - [[ -d /sys/bus/pci/drivers/mlx5_core ]] + $maximize_crashkernel || [[ -d /sys/bus/pci/drivers/mlx5_core ]] } has_aarch64_smmu() { - ls /sys/devices/platform/arm-smmu-* 1> /dev/null 2>&1 + $maximize_crashkernel || ls /sys/devices/platform/arm-smmu-* 1> /dev/null 2>&1 +} + +is_aarch64_64k_kernel() +{ + local _kernel="$1" + # the naming convention of 64k variant suffixes with +64k, e.g. "vmlinuz-5.14.0-312.el9.aarch64+64k" + $maximize_crashkernel || echo "$_kernel" | grep -q 64k } is_memsize() @@ -1010,6 +1018,9 @@ kdump_get_arch_recommend_crashkernel() local _delta=0 local _skip=0 + # osbuild deploys rpm on isolated environment. kdump-utils has no opportunity + # to deduce the exact memory cost on the real target. + _is_osbuild && maximize_crashkernel=true if [[ -z $1 ]]; then if is_fadump_capable; then _dump_mode=fadump @@ -1038,8 +1049,7 @@ kdump_get_arch_recommend_crashkernel() # skip adding additional memory for small-memory machine _skip=1 - # the naming convention of 64k variant suffixes with +64k, e.g. "vmlinuz-5.14.0-312.el9.aarch64+64k" - if echo "$_running_kernel" | grep -q 64k; then + if is_aarch64_64k_kernel "$_running_kernel"; then # Without smmu, the diff of MemFree between 4K and 64K measured on a high end aarch64 machine is 82M. # Picking up 100M to cover this diff. And finally, we have "2G-4G:356M;4G-64G:420M;64G-:676M" ((_delta += 100)) From 7602d662e15cdfea7fc36f0b7fb977277bbd0d59 Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Tue, 24 Jun 2025 10:15:18 +0800 Subject: [PATCH 7/7] kdumpctl: Check the recommend value when starting service The crashkernel value is maximized for a OSTree image. When the image runs on the final platform, kdump-utils has the chance to get the proper reserved memory size and prompts the user to update the crashkernel parameter. Signed-off-by: Pingfan Liu --- kdumpctl | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/kdumpctl b/kdumpctl index 6d5b3b8b..d148ab65 100755 --- a/kdumpctl +++ b/kdumpctl @@ -1210,6 +1210,53 @@ prepare_luks() done } +get_crashkernel_bounce_buffer_size() +{ + local lines first_line start_hex end_hex start_dec end_dec size_bytes + + mapfile -t lines < <(grep "Crash kernel" /proc/iomem) + + if [ ${#lines[@]} -le 1 ]; then + echo 0 + return + fi + + first_line=${lines[0]} + # Extract start and end addresses + start_hex=$(echo "$first_line" | awk '{split($1, a, "-"); print a[1]}') + end_hex=$(echo "$first_line" | awk '{split($1, a, "-"); print a[2]}') + # Convert hex to decimal + start_dec=$((0x$start_hex)) + end_dec=$((0x$end_hex)) + # Calculate size in bytes + size_bytes=$((end_dec - start_dec + 1)) + echo "$size_bytes" +} + +suggest_crashkernel_reset() +{ + local _crashkernel _recommend_value _actual_value _bounce_buff_size + + _crashkernel=$(sed -n 's/.*crashkernel=\([^ ]*\).*/\1/p' /proc/cmdline) + [[ $(kdump_get_conf_val auto_reset_crashkernel) == no ]] && return + # At present, the crashkernel value for fadump is not dynamic + is_fadump_capable && return + + _recommend_value=$(kdump_get_arch_recommend_size) + # The crashkernel formula does not account for DMA-bounce buffers, + # unlike kexec_crash_size which does. Systems using DMA-bounce buffers + # should factor this into the calculation. + _bounce_buff_size=$(get_crashkernel_bounce_buffer_size) + _recommend_value=$(memsize_add "$_recommend_value" "$_bounce_buff_size") + _recommend_value=$(to_bytes "$_recommend_value") + _actual_value=$(cat /sys/kernel/kexec_crash_size) + + if [ "$_recommend_value" -lt "$_actual_value" ]; then + dwarn "The reserved crashkernel is abundant. Using 'kdumpctl reset-crashkernel' to reset kernel cmdline. It will take effect in the next boot" + dwarn "To release the abundant memory immediately, you can run: 'kdumpctl stop; echo $_recommend_value >/sys/kernel/kexec_crash_size; kdumpctl start'" + fi +} + start() { check_dump_feasibility || return @@ -1243,6 +1290,7 @@ start() start_dump || return dinfo "Starting kdump: [OK]" + suggest_crashkernel_reset return 0 }