From 807ab01e3857c9b907296643c448d535eedf875e Mon Sep 17 00:00:00 2001 From: Chasing1020 Date: Wed, 27 May 2026 13:41:04 +0000 Subject: [PATCH 1/2] Fix PYTHONBUFFERED typo to PYTHONUNBUFFERED=1 The intended env var to disable Python stdout/stderr buffering is PYTHONUNBUFFERED, not PYTHONBUFFERED. The misspelled variable had no effect, so Ray/training stdout was still subject to buffering. --- docker/npu_patch/slime.patch | 4 ++-- docs/en/platform_support/amd_tutorial.md | 2 +- examples/delta_weight_sync/run-glm4.7-355B-A32B-delta.sh | 2 +- examples/eval_multi_task/multi_task.sh | 2 +- examples/fully_async/run-qwen2.5-0.5B-fully_async.sh | 2 +- examples/geo3k_vlm/run_geo3k_qwen35.sh | 2 +- examples/geo3k_vlm/run_geo3k_vlm.sh | 2 +- examples/geo3k_vlm/run_geo3k_vlm_sft.sh | 2 +- examples/multi_agent/run-qwen3-30B-A3B-multi-agent.sh | 2 +- examples/on_policy_distillation/run-qwen3-8B-opd-megatron.sh | 2 +- examples/on_policy_distillation/run-qwen3-8B-opd.sh | 2 +- examples/retool/retool_qwen3_4b_rl.sh | 2 +- examples/retool/retool_qwen3_4b_sft.sh | 2 +- examples/search-r1/run_qwen2.5_3B.sh | 2 +- examples/strands_sglang/strands_qwen3_8b.sh | 2 +- examples/tau-bench/run_qwen3_4B.sh | 2 +- examples/train_infer_mismatch_helper/run-qwen3-4b-mis.sh | 2 +- scripts/low_precision/run-kimi-k2-Thinking-int4.sh | 2 +- scripts/low_precision/run-moonlight-16B-A3B-int4.sh | 2 +- scripts/low_precision/run-qwen3-235B-A22B-int4.sh | 2 +- scripts/low_precision/run-qwen3-30B-A3B-int4.sh | 2 +- scripts/low_precision/run-qwen3-30b-a3b-fp8.sh | 2 +- scripts/low_precision/run-qwen3-4b-fp8.sh | 2 +- scripts/run-deepseek-r1.sh | 2 +- scripts/run-glm4-9B.sh | 2 +- scripts/run-glm4.7-30B-A3B.sh | 2 +- scripts/run-glm4.7-355B-A32B.sh | 2 +- scripts/run-glm5-744B-A40B.sh | 2 +- scripts/run-gpt-oss-20B.sh | 2 +- scripts/run-kimi-k2-Instruct.sh | 2 +- scripts/run-kimi-k2-Thinking.sh | 2 +- scripts/run-mimo-7B-rl-eagle.sh | 2 +- scripts/run-moonlight-16B-A3B.sh | 2 +- scripts/run-qwen2.5-0.5B-gb10-smoke.sh | 2 +- scripts/run-qwen2.5-0.5B-reproducibility.sh | 2 +- scripts/run-qwen3-235B-A22B-sft.sh | 2 +- scripts/run-qwen3-235B-A22B.sh | 2 +- scripts/run-qwen3-30B-A3B.sh | 2 +- scripts/run-qwen3-32B.sh | 2 +- scripts/run-qwen3-4B-amd.sh | 2 +- scripts/run-qwen3-4B-base-sft.sh | 2 +- scripts/run-qwen3-4B.sh | 2 +- scripts/run-qwen3-next-80B-A3B.sh | 2 +- scripts/run-qwen3.5-27B.sh | 2 +- scripts/run-qwen3.5-35B-A3B-sft.sh | 2 +- slime/utils/external_utils/command_utils.py | 4 ++-- slime_plugins/rollout_buffer/rollout_buffer_example.sh | 2 +- tests/test_gspo.sh | 2 +- 48 files changed, 50 insertions(+), 50 deletions(-) diff --git a/docker/npu_patch/slime.patch b/docker/npu_patch/slime.patch index 55e7c2d44e..ebb6c000b5 100644 --- a/docker/npu_patch/slime.patch +++ b/docker/npu_patch/slime.patch @@ -692,7 +692,7 @@ index 9f51ecdf..d4b47eca 100644 + if not external_ray: + exec_command( + # will prevent ray from buffering stdout/stderr -+ f"export PYTHONBUFFERED=16 && " ++ f"export PYTHONUNBUFFERED=1 && " + f"ray start --head --node-ip-address {master_addr} --disable-usage-stats" + ) + @@ -748,7 +748,7 @@ index 9f51ecdf..d4b47eca 100644 + else "" + ) + exec_command( -+ f"export no_proxy=127.0.0.1 && export PYTHONBUFFERED=16 && " ++ f"export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && " + f"{cmd_megatron_model_source}" + f'ray job submit --address="http://127.0.0.1:8265" ' + f"--runtime-env-json='{runtime_env_json}' " diff --git a/docs/en/platform_support/amd_tutorial.md b/docs/en/platform_support/amd_tutorial.md index 73d690fb18..36fb40c1a2 100644 --- a/docs/en/platform_support/amd_tutorial.md +++ b/docs/en/platform_support/amd_tutorial.md @@ -148,7 +148,7 @@ export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-"0,1,2,3,4,5,6,7"} #You can ch # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${SCRIPT_DIR}/models/qwen3-4B.sh" diff --git a/examples/delta_weight_sync/run-glm4.7-355B-A32B-delta.sh b/examples/delta_weight_sync/run-glm4.7-355B-A32B-delta.sh index 9cf2dc9246..68f35bb643 100755 --- a/examples/delta_weight_sync/run-glm4.7-355B-A32B-delta.sh +++ b/examples/delta_weight_sync/run-glm4.7-355B-A32B-delta.sh @@ -15,7 +15,7 @@ pkill -9 python set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) diff --git a/examples/eval_multi_task/multi_task.sh b/examples/eval_multi_task/multi_task.sh index 01b884110a..8d4fe1994c 100644 --- a/examples/eval_multi_task/multi_task.sh +++ b/examples/eval_multi_task/multi_task.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/fully_async/run-qwen2.5-0.5B-fully_async.sh b/examples/fully_async/run-qwen2.5-0.5B-fully_async.sh index f843f28501..25e134438c 100755 --- a/examples/fully_async/run-qwen2.5-0.5B-fully_async.sh +++ b/examples/fully_async/run-qwen2.5-0.5B-fully_async.sh @@ -17,7 +17,7 @@ sleep 3 set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) HAS_NVLINK=$([ "$NVLINK_COUNT" -gt 0 ] && echo 1 || echo 0) diff --git a/examples/geo3k_vlm/run_geo3k_qwen35.sh b/examples/geo3k_vlm/run_geo3k_qwen35.sh index 8b402c8dfd..8f057de887 100644 --- a/examples/geo3k_vlm/run_geo3k_qwen35.sh +++ b/examples/geo3k_vlm/run_geo3k_qwen35.sh @@ -44,7 +44,7 @@ pkill -9 redis set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 # Detect NVLink NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) diff --git a/examples/geo3k_vlm/run_geo3k_vlm.sh b/examples/geo3k_vlm/run_geo3k_vlm.sh index 098a329906..602e4ae16a 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm.sh @@ -59,7 +59,7 @@ pkill -9 redis set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 # Detect NVLink NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) diff --git a/examples/geo3k_vlm/run_geo3k_vlm_sft.sh b/examples/geo3k_vlm/run_geo3k_vlm_sft.sh index 06b16d5f92..678998f605 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm_sft.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm_sft.sh @@ -52,7 +52,7 @@ pkill -9 redis set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 # Detect NVLink NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) diff --git a/examples/multi_agent/run-qwen3-30B-A3B-multi-agent.sh b/examples/multi_agent/run-qwen3-30B-A3B-multi-agent.sh index a9e8b9c04a..d41f6bec21 100644 --- a/examples/multi_agent/run-qwen3-30B-A3B-multi-agent.sh +++ b/examples/multi_agent/run-qwen3-30B-A3B-multi-agent.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/on_policy_distillation/run-qwen3-8B-opd-megatron.sh b/examples/on_policy_distillation/run-qwen3-8B-opd-megatron.sh index 8f64bfab0b..2f798a7550 100644 --- a/examples/on_policy_distillation/run-qwen3-8B-opd-megatron.sh +++ b/examples/on_policy_distillation/run-qwen3-8B-opd-megatron.sh @@ -11,7 +11,7 @@ set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/on_policy_distillation/run-qwen3-8B-opd.sh b/examples/on_policy_distillation/run-qwen3-8B-opd.sh index 386a3e3662..3fdc479b16 100644 --- a/examples/on_policy_distillation/run-qwen3-8B-opd.sh +++ b/examples/on_policy_distillation/run-qwen3-8B-opd.sh @@ -34,7 +34,7 @@ echo "Teacher model server is up and running at $TEACHER_IP:$TEACHER_PORT." sleep 10 -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/retool/retool_qwen3_4b_rl.sh b/examples/retool/retool_qwen3_4b_rl.sh index ebb54ad6bb..32a837f394 100644 --- a/examples/retool/retool_qwen3_4b_rl.sh +++ b/examples/retool/retool_qwen3_4b_rl.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/retool/retool_qwen3_4b_sft.sh b/examples/retool/retool_qwen3_4b_sft.sh index a489a58104..574c9460bf 100644 --- a/examples/retool/retool_qwen3_4b_sft.sh +++ b/examples/retool/retool_qwen3_4b_sft.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/search-r1/run_qwen2.5_3B.sh b/examples/search-r1/run_qwen2.5_3B.sh index d5acfa4563..2f2fdc444d 100644 --- a/examples/search-r1/run_qwen2.5_3B.sh +++ b/examples/search-r1/run_qwen2.5_3B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${SCRIPT_DIR}/../../scripts/models/qwen2.5-3B.sh" diff --git a/examples/strands_sglang/strands_qwen3_8b.sh b/examples/strands_sglang/strands_qwen3_8b.sh index c61d7d349e..5f7af4f751 100644 --- a/examples/strands_sglang/strands_qwen3_8b.sh +++ b/examples/strands_sglang/strands_qwen3_8b.sh @@ -16,7 +16,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/tau-bench/run_qwen3_4B.sh b/examples/tau-bench/run_qwen3_4B.sh index a821734012..12bdb270ed 100644 --- a/examples/tau-bench/run_qwen3_4B.sh +++ b/examples/tau-bench/run_qwen3_4B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/examples/train_infer_mismatch_helper/run-qwen3-4b-mis.sh b/examples/train_infer_mismatch_helper/run-qwen3-4b-mis.sh index dd8d2bed6c..14a7011e03 100644 --- a/examples/train_infer_mismatch_helper/run-qwen3-4b-mis.sh +++ b/examples/train_infer_mismatch_helper/run-qwen3-4b-mis.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/low_precision/run-kimi-k2-Thinking-int4.sh b/scripts/low_precision/run-kimi-k2-Thinking-int4.sh index f7abd62b57..7a89241a47 100644 --- a/scripts/low_precision/run-kimi-k2-Thinking-int4.sh +++ b/scripts/low_precision/run-kimi-k2-Thinking-int4.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi | grep -o "NVLink" | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/low_precision/run-moonlight-16B-A3B-int4.sh b/scripts/low_precision/run-moonlight-16B-A3B-int4.sh index 9015e577e9..55026c9999 100644 --- a/scripts/low_precision/run-moonlight-16B-A3B-int4.sh +++ b/scripts/low_precision/run-moonlight-16B-A3B-int4.sh @@ -14,7 +14,7 @@ pkill -9 redis set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/low_precision/run-qwen3-235B-A22B-int4.sh b/scripts/low_precision/run-qwen3-235B-A22B-int4.sh index b5ddc7587c..abcb2a97d7 100644 --- a/scripts/low_precision/run-qwen3-235B-A22B-int4.sh +++ b/scripts/low_precision/run-qwen3-235B-A22B-int4.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi | grep -o "NVLink" | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/low_precision/run-qwen3-30B-A3B-int4.sh b/scripts/low_precision/run-qwen3-30B-A3B-int4.sh index b591047e10..018ef2da7e 100644 --- a/scripts/low_precision/run-qwen3-30B-A3B-int4.sh +++ b/scripts/low_precision/run-qwen3-30B-A3B-int4.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderrs -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/low_precision/run-qwen3-30b-a3b-fp8.sh b/scripts/low_precision/run-qwen3-30b-a3b-fp8.sh index 766e0dcc62..cdce4a3b1b 100644 --- a/scripts/low_precision/run-qwen3-30b-a3b-fp8.sh +++ b/scripts/low_precision/run-qwen3-30b-a3b-fp8.sh @@ -14,7 +14,7 @@ set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/low_precision/run-qwen3-4b-fp8.sh b/scripts/low_precision/run-qwen3-4b-fp8.sh index 64a1487e17..eccf2872d8 100644 --- a/scripts/low_precision/run-qwen3-4b-fp8.sh +++ b/scripts/low_precision/run-qwen3-4b-fp8.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-deepseek-r1.sh b/scripts/run-deepseek-r1.sh index f270c52f8e..b2a4d824e9 100644 --- a/scripts/run-deepseek-r1.sh +++ b/scripts/run-deepseek-r1.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-glm4-9B.sh b/scripts/run-glm4-9B.sh index 8462cb2f99..72e67d0866 100644 --- a/scripts/run-glm4-9B.sh +++ b/scripts/run-glm4-9B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-glm4.7-30B-A3B.sh b/scripts/run-glm4.7-30B-A3B.sh index 1b52b25460..bec6bfc492 100644 --- a/scripts/run-glm4.7-30B-A3B.sh +++ b/scripts/run-glm4.7-30B-A3B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY diff --git a/scripts/run-glm4.7-355B-A32B.sh b/scripts/run-glm4.7-355B-A32B.sh index aea94f4b6d..f9e98d7fb9 100644 --- a/scripts/run-glm4.7-355B-A32B.sh +++ b/scripts/run-glm4.7-355B-A32B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY diff --git a/scripts/run-glm5-744B-A40B.sh b/scripts/run-glm5-744B-A40B.sh index 08e624b267..63c6601c3c 100644 --- a/scripts/run-glm5-744B-A40B.sh +++ b/scripts/run-glm5-744B-A40B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-gpt-oss-20B.sh b/scripts/run-gpt-oss-20B.sh index 0fdcd82ae0..c576b22f05 100755 --- a/scripts/run-gpt-oss-20B.sh +++ b/scripts/run-gpt-oss-20B.sh @@ -23,7 +23,7 @@ pkill -9 python set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-kimi-k2-Instruct.sh b/scripts/run-kimi-k2-Instruct.sh index 3a591b923a..6ebf324c3c 100644 --- a/scripts/run-kimi-k2-Instruct.sh +++ b/scripts/run-kimi-k2-Instruct.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-kimi-k2-Thinking.sh b/scripts/run-kimi-k2-Thinking.sh index 25cc3c475f..e373d09a78 100644 --- a/scripts/run-kimi-k2-Thinking.sh +++ b/scripts/run-kimi-k2-Thinking.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-mimo-7B-rl-eagle.sh b/scripts/run-mimo-7B-rl-eagle.sh index 9e65fd2cd2..ed631c77b6 100644 --- a/scripts/run-mimo-7B-rl-eagle.sh +++ b/scripts/run-mimo-7B-rl-eagle.sh @@ -14,7 +14,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-moonlight-16B-A3B.sh b/scripts/run-moonlight-16B-A3B.sh index ec54c18326..6e1f5965c1 100644 --- a/scripts/run-moonlight-16B-A3B.sh +++ b/scripts/run-moonlight-16B-A3B.sh @@ -14,7 +14,7 @@ pkill -9 redis set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-qwen2.5-0.5B-gb10-smoke.sh b/scripts/run-qwen2.5-0.5B-gb10-smoke.sh index 294c6836fc..7a379b9614 100755 --- a/scripts/run-qwen2.5-0.5B-gb10-smoke.sh +++ b/scripts/run-qwen2.5-0.5B-gb10-smoke.sh @@ -17,7 +17,7 @@ ray stop --force 2>/dev/null || true pkill -9 ray python 2>/dev/null || true sleep 2 -export PYTHONBUFFERED=1 +export PYTHONUNBUFFERED=1 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${SCRIPT_DIR}/models/qwen2.5-0.5B.sh" diff --git a/scripts/run-qwen2.5-0.5B-reproducibility.sh b/scripts/run-qwen2.5-0.5B-reproducibility.sh index d1d2d3bf1c..446eca65e1 100644 --- a/scripts/run-qwen2.5-0.5B-reproducibility.sh +++ b/scripts/run-qwen2.5-0.5B-reproducibility.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${SCRIPT_DIR}/scripts/models/qwen2.5-0.5B.sh" diff --git a/scripts/run-qwen3-235B-A22B-sft.sh b/scripts/run-qwen3-235B-A22B-sft.sh index 744fbc7921..14ac8e4a6c 100644 --- a/scripts/run-qwen3-235B-A22B-sft.sh +++ b/scripts/run-qwen3-235B-A22B-sft.sh @@ -24,7 +24,7 @@ if [ -z "${MASTER_ADDR}" ]; then fi # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-qwen3-235B-A22B.sh b/scripts/run-qwen3-235B-A22B.sh index 28ef17a221..679c4172e3 100644 --- a/scripts/run-qwen3-235B-A22B.sh +++ b/scripts/run-qwen3-235B-A22B.sh @@ -24,7 +24,7 @@ if [ -z "${MASTER_ADDR}" ]; then fi # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-qwen3-30B-A3B.sh b/scripts/run-qwen3-30B-A3B.sh index 2b8686af98..b2d01df6d3 100644 --- a/scripts/run-qwen3-30B-A3B.sh +++ b/scripts/run-qwen3-30B-A3B.sh @@ -14,7 +14,7 @@ pkill -9 redis set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-qwen3-32B.sh b/scripts/run-qwen3-32B.sh index 88c3ddcbce..4d870ca391 100644 --- a/scripts/run-qwen3-32B.sh +++ b/scripts/run-qwen3-32B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-qwen3-4B-amd.sh b/scripts/run-qwen3-4B-amd.sh index 8452d8e5f8..d31abacdf0 100755 --- a/scripts/run-qwen3-4B-amd.sh +++ b/scripts/run-qwen3-4B-amd.sh @@ -31,7 +31,7 @@ export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-"0,1,2,3,4,5,6,7"} #You can ch # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${SCRIPT_DIR}/models/qwen3-4B.sh" diff --git a/scripts/run-qwen3-4B-base-sft.sh b/scripts/run-qwen3-4B-base-sft.sh index 7bfd061b19..6860781d30 100644 --- a/scripts/run-qwen3-4B-base-sft.sh +++ b/scripts/run-qwen3-4B-base-sft.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-qwen3-4B.sh b/scripts/run-qwen3-4B.sh index 17023da848..ec06a8f525 100644 --- a/scripts/run-qwen3-4B.sh +++ b/scripts/run-qwen3-4B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/scripts/run-qwen3-next-80B-A3B.sh b/scripts/run-qwen3-next-80B-A3B.sh index 1b445c534d..9347b39806 100644 --- a/scripts/run-qwen3-next-80B-A3B.sh +++ b/scripts/run-qwen3-next-80B-A3B.sh @@ -24,7 +24,7 @@ if [ -z "${MASTER_ADDR}" ]; then fi # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 # unset proxy to avoid distributed startup issues unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY diff --git a/scripts/run-qwen3.5-27B.sh b/scripts/run-qwen3.5-27B.sh index 3770850034..28d4ba2185 100755 --- a/scripts/run-qwen3.5-27B.sh +++ b/scripts/run-qwen3.5-27B.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 # unset proxy to avoid distributed startup issues unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY diff --git a/scripts/run-qwen3.5-35B-A3B-sft.sh b/scripts/run-qwen3.5-35B-A3B-sft.sh index 6893133924..71692c1f6a 100644 --- a/scripts/run-qwen3.5-35B-A3B-sft.sh +++ b/scripts/run-qwen3.5-35B-A3B-sft.sh @@ -25,7 +25,7 @@ fi # export MASTER_ADDR="127.0.0.1" # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) if [ "$NVLINK_COUNT" -gt 0 ]; then diff --git a/slime/utils/external_utils/command_utils.py b/slime/utils/external_utils/command_utils.py index cdf2cbe0b0..af32de59fe 100644 --- a/slime/utils/external_utils/command_utils.py +++ b/slime/utils/external_utils/command_utils.py @@ -127,7 +127,7 @@ def execute_train( if not external_ray: exec_command( # will prevent ray from buffering stdout/stderr - f"export PYTHONBUFFERED=16 && " + f"export PYTHONUNBUFFERED=1 && " f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus_per_node} --disable-usage-stats" ) @@ -166,7 +166,7 @@ def execute_train( else "" ) exec_command( - f"export no_proxy=127.0.0.1 && export PYTHONBUFFERED=16 && " + f"export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && " f"{cmd_megatron_model_source}" f'ray job submit --address="http://127.0.0.1:8265" ' f"--runtime-env-json='{runtime_env_json}' " diff --git a/slime_plugins/rollout_buffer/rollout_buffer_example.sh b/slime_plugins/rollout_buffer/rollout_buffer_example.sh index c0443d7d72..ff51e45be1 100644 --- a/slime_plugins/rollout_buffer/rollout_buffer_example.sh +++ b/slime_plugins/rollout_buffer/rollout_buffer_example.sh @@ -12,7 +12,7 @@ pkill -9 python set -ex -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 # DeepSeek-R1-Distill-Qwen-7B MODEL_ARGS=( diff --git a/tests/test_gspo.sh b/tests/test_gspo.sh index d26a479038..2ea971c73a 100644 --- a/tests/test_gspo.sh +++ b/tests/test_gspo.sh @@ -13,7 +13,7 @@ pkill -9 python set -ex # will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 CKPT_ARGS=( --hf-checkpoint /root/Qwen3-0.6B From 9377d0ab2a7b55576edc9fae34ca1a292eb0e181 Mon Sep 17 00:00:00 2001 From: Jiancong Zhu Date: Wed, 27 May 2026 22:45:48 +0800 Subject: [PATCH 2/2] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- scripts/low_precision/run-qwen3-30B-A3B-int4.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/low_precision/run-qwen3-30B-A3B-int4.sh b/scripts/low_precision/run-qwen3-30B-A3B-int4.sh index 018ef2da7e..1770eef9f9 100644 --- a/scripts/low_precision/run-qwen3-30B-A3B-int4.sh +++ b/scripts/low_precision/run-qwen3-30B-A3B-int4.sh @@ -12,7 +12,7 @@ pkill -9 python set -ex -# will prevent ray from buffering stdout/stderrs +# will prevent ray from buffering stdout/stderr export PYTHONUNBUFFERED=1 NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l)