From 6d3515b55cf7194467823c85765ec35d7340e841 Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Mon, 9 Mar 2026 23:22:00 +0800 Subject: [PATCH 01/11] bugfix: prevent uthread crash by checking coroutine reference before deletion --- src/ngx_http_lua_util.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 2dc4f84902..afe4c9ab34 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -1425,8 +1425,10 @@ ngx_http_lua_run_thread(lua_State *L, ngx_http_request_t *r, return NGX_AGAIN; } - ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); - ctx->uthreads--; + if (ctx->cur_co_ctx->co_ref != LUA_NOREF) { + ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); + ctx->uthreads--; + } if (ctx->uthreads == 0) { if (ngx_http_lua_entry_thread_alive(ctx)) { @@ -1464,7 +1466,8 @@ ngx_http_lua_run_thread(lua_State *L, ngx_http_request_t *r, lua_xmove(ctx->cur_co_ctx->co, next_co, nrets); } - if (ctx->cur_co_ctx->is_uthread) { + if (ctx->cur_co_ctx->is_uthread + && ctx->cur_co_ctx->co_ref != LUA_NOREF) ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); ctx->uthreads--; } @@ -1575,8 +1578,10 @@ ngx_http_lua_run_thread(lua_State *L, ngx_http_request_t *r, return NGX_AGAIN; } - ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); - ctx->uthreads--; + if (ctx->cur_co_ctx->co_ref != LUA_NOREF) { + ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); + ctx->uthreads--; + } if (ctx->uthreads == 0) { if (ngx_http_lua_entry_thread_alive(ctx)) { From 1f68babe2965cac33c897b8d00eed1d9e2ac29f5 Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Mon, 9 Mar 2026 23:27:47 +0800 Subject: [PATCH 02/11] fix --- t/127-uthread-kill.t | 110 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/t/127-uthread-kill.t b/t/127-uthread-kill.t index 11caee582f..8d3b503357 100644 --- a/t/127-uthread-kill.t +++ b/t/127-uthread-kill.t @@ -12,6 +12,7 @@ plan tests => repeat_each() * (blocks() * 5 + 1); $ENV{TEST_NGINX_RESOLVER} ||= '8.8.8.8'; $ENV{TEST_NGINX_MEMCACHED_PORT} ||= '11211'; +$ENV{TEST_NGINX_REDIS_PORT} ||= '6379'; #no_shuffle(); no_long_string(); @@ -505,3 +506,112 @@ thread created: zombie [alert] lua tcp socket abort resolver --- error_log + +=== TEST 10: phantom uthreads-- via killed uthread with live child coroutine +When a user thread internally uses coroutine.resume() to run a child +coroutine that does cosocket I/O, and the parent uthread is then killed +via ngx.thread.kill(), the child's socket I/O is NOT cancelled (because +cleanup_pending_operation on the parent's co_ctx is a no-op -- the parent +yielded for coroutine.resume, not for I/O, so cleanup is NULL). + +When the child later completes, user_co_done walks up to the killed +parent's co_ctx, finds is_uthread=1, and does uthreads-- even though +del_thread returns early (co_ref already LUA_NOREF). This phantom +decrement causes uthreads to underflow when finalize_threads runs +during the access->content phase transition. +--- config + location = /t { + rewrite_by_lua_block { + ngx.ctx._rewrite = true + } + + access_by_lua_block { + local redis_port = $TEST_NGINX_REDIS_PORT + + -- Phase 1: simulate DNS resolution with thread.spawn + wait + kill + -- (like thread_process_dns_query in resolver.lua) + local dns_threads = {} + + -- T1: fast DNS query (PING, completes immediately) + dns_threads[1] = ngx.thread.spawn(function() + local sock = ngx.socket.tcp() + sock:settimeout(2000) + local ok, err = sock:connect("127.0.0.1", redis_port) + if not ok then return nil, err end + sock:send("PING\r\n") + local line = sock:receive() + sock:setkeepalive() + return line + end) + + -- T2: uses coroutine.resume internally with FAST I/O. + -- The child coroutine does a fast Redis PING so its response + -- arrives in the same epoll batch as T1's response. + -- When T2 is killed, the child's socket is NOT cleaned up + -- because T2 yielded for coroutine.resume (cleanup=NULL), + -- not for socket I/O. The child's event then fires AFTER + -- T2 is killed but BEFORE finalize_threads runs. + dns_threads[2] = ngx.thread.spawn(function() + local child = coroutine.create(function() + local sock = ngx.socket.tcp() + sock:settimeout(2000) + local ok, err = sock:connect("127.0.0.1", redis_port) + if not ok then return nil, err end + sock:send("PING\r\n") + local line = sock:receive() + sock:setkeepalive() + return line + end) + + local ok, res = coroutine.resume(child) + return res + end) + + -- wait_any pattern: wait for first result + local ok, res = ngx.thread.wait(dns_threads[1], dns_threads[2]) + + -- kill remaining (T2's child socket is still alive!) + for _, t in ipairs(dns_threads) do + ngx.thread.kill(t) + end + + -- Phase 2: simulate probe thread spawning + -- (like run_batch_probe_targets in edge_probe) + local probe_threads = {} + for i = 1, 10 do + probe_threads[i] = ngx.thread.spawn(function(idx) + local sock = ngx.socket.tcp() + sock:settimeout(2000) + local ok, err = sock:connect("127.0.0.1", redis_port) + if not ok then return nil, err end + sock:send("PING\r\n") + local line = sock:receive() + sock:setkeepalive() + return line + end, i) + end + + local ok_count = 0 + for i = 1, #probe_threads do + local ok, res, err = ngx.thread.wait(probe_threads[i]) + if ok and res then + ok_count = ok_count + 1 + end + end + + ngx.ctx.ok_count = ok_count + } + + content_by_lua_block { + -- reset_ctx -> finalize_threads runs here. + -- Without the fix, the phantom uthreads-- from the killed T2's + -- child completing causes uthreads underflow. + ngx.say("ok_count=", ngx.ctx.ok_count) + } + } +--- request +GET /t +--- response_body +ok_count=10 +--- no_error_log +[error] From 9f0b5a382fd915d503a0745b19668bb587d8157c Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Tue, 10 Mar 2026 11:32:37 +0800 Subject: [PATCH 03/11] fix --- t/127-uthread-kill.t | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t/127-uthread-kill.t b/t/127-uthread-kill.t index 8d3b503357..1467b2e286 100644 --- a/t/127-uthread-kill.t +++ b/t/127-uthread-kill.t @@ -507,6 +507,8 @@ thread created: zombie lua tcp socket abort resolver --- error_log + + === TEST 10: phantom uthreads-- via killed uthread with live child coroutine When a user thread internally uses coroutine.resume() to run a child coroutine that does cosocket I/O, and the parent uthread is then killed From 6772f993241e39c1153fe1d3cab8e520605956fd Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Tue, 10 Mar 2026 13:20:44 +0800 Subject: [PATCH 04/11] fix --- t/127-uthread-kill.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/127-uthread-kill.t b/t/127-uthread-kill.t index 1467b2e286..fc2c376eca 100644 --- a/t/127-uthread-kill.t +++ b/t/127-uthread-kill.t @@ -8,7 +8,7 @@ our $StapScript = $t::StapThread::StapScript; repeat_each(2); -plan tests => repeat_each() * (blocks() * 5 + 1); +plan tests => repeat_each() * (blocks() * 5 + 1) -2; $ENV{TEST_NGINX_RESOLVER} ||= '8.8.8.8'; $ENV{TEST_NGINX_MEMCACHED_PORT} ||= '11211'; From db0fce0179c70c25ef155b0a6f11e7641e1f15cb Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Tue, 10 Mar 2026 13:40:48 +0800 Subject: [PATCH 05/11] fix --- src/ngx_http_lua_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index afe4c9ab34..66767f8689 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -1467,7 +1467,7 @@ ngx_http_lua_run_thread(lua_State *L, ngx_http_request_t *r, } if (ctx->cur_co_ctx->is_uthread - && ctx->cur_co_ctx->co_ref != LUA_NOREF) + && ctx->cur_co_ctx->co_ref != LUA_NOREF) { ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); ctx->uthreads--; } From cbe99ec1b90e3c5fa3e7bd2a3fd4c1da8d0e6f4e Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Tue, 10 Mar 2026 13:46:17 +0800 Subject: [PATCH 06/11] fix --- src/ngx_http_lua_util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 66767f8689..09e0411cb9 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -1468,8 +1468,8 @@ ngx_http_lua_run_thread(lua_State *L, ngx_http_request_t *r, if (ctx->cur_co_ctx->is_uthread && ctx->cur_co_ctx->co_ref != LUA_NOREF) { - ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); - ctx->uthreads--; + ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); + ctx->uthreads--; } if (!ctx->cur_co_ctx->is_wrap) { From 377a945004cc7c6b954f5c84213312007c244322 Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Tue, 10 Mar 2026 13:58:41 +0800 Subject: [PATCH 07/11] fix --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ee5f43d1c7..d119ba9461 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,7 +86,7 @@ install: #- if [ -n "$OPENSSL_VER" ] && [ ! -f download-cache/openssl-$OPENSSL_VER.tar.gz ]; then wget -P download-cache https://github.com/openssl/openssl/releases/download/openssl-$OPENSSL_VER/openssl-$OPENSSL_VER.tar.gz || wget -P download-cache https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz || wget -P download-cache https://www.openssl.org/source/old/${OPENSSL_VER//[a-z]/}/openssl-$OPENSSL_VER.tar.gz; fi - wget https://github.com/openresty/openresty-deps-prebuild/releases/download/v20230902/boringssl-20230902-x64-focal.tar.gz - wget https://github.com/openresty/openresty-deps-prebuild/releases/download/v20230902/curl-h3-x64-focal.tar.gz - - git clone https://github.com/openresty/test-nginx.git + - git clone --revision=efb42ee915b079b6b5e7fb24ee2afd7be44fa2bf --depth=1 https://github.com/openresty/test-nginx.git - git clone https://github.com/openresty/openresty.git ../openresty - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx - git clone https://github.com/openresty/openresty-devel-utils.git From 172891e128a07a43ac8d7a603dfca81acb24f621 Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Tue, 10 Mar 2026 14:23:34 +0800 Subject: [PATCH 08/11] Adjust test plan count in 127-uthread-kill.t --- t/127-uthread-kill.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/127-uthread-kill.t b/t/127-uthread-kill.t index fc2c376eca..77cf2bf9d3 100644 --- a/t/127-uthread-kill.t +++ b/t/127-uthread-kill.t @@ -8,7 +8,7 @@ our $StapScript = $t::StapThread::StapScript; repeat_each(2); -plan tests => repeat_each() * (blocks() * 5 + 1) -2; +plan tests => repeat_each() * (blocks() * 5 + 1) - 4; $ENV{TEST_NGINX_RESOLVER} ||= '8.8.8.8'; $ENV{TEST_NGINX_MEMCACHED_PORT} ||= '11211'; From e913b7dcd68a0cbd18c540c911f1495ef8df09b7 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Tue, 10 Mar 2026 13:44:55 +0800 Subject: [PATCH 09/11] fixed style --- src/ngx_http_lua_util.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 09e0411cb9..01c36440e5 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -1467,9 +1467,10 @@ ngx_http_lua_run_thread(lua_State *L, ngx_http_request_t *r, } if (ctx->cur_co_ctx->is_uthread - && ctx->cur_co_ctx->co_ref != LUA_NOREF) { - ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); - ctx->uthreads--; + && ctx->cur_co_ctx->co_ref != LUA_NOREF) + { + ngx_http_lua_del_thread(r, L, ctx, ctx->cur_co_ctx); + ctx->uthreads--; } if (!ctx->cur_co_ctx->is_wrap) { From 73fedbd24498770b5aa8500bbcc4c852e00ec5fb Mon Sep 17 00:00:00 2001 From: lijunlong Date: Tue, 10 Mar 2026 14:24:56 +0800 Subject: [PATCH 10/11] more fixes --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d119ba9461..c80235a4ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ services: before_install: - '! grep -n -P ''(?<=.{80}).+'' --color `find src -name ''*.c''` `find . -name ''*.h''` || (echo "ERROR: Found C source lines exceeding 80 columns." > /dev/stderr; exit 1)' - '! grep -n -P ''\t+'' --color `find src -name ''*.c''` `find . -name ''*.h''` || (echo "ERROR: Cannot use tabs." > /dev/stderr; exit 1)' - - /usr/bin/env perl $(command -v cpanm) --sudo --notest Test::Nginx IPC::Run > build.log 2>&1 || (cat build.log && exit 1) + - /usr/bin/env perl $(command -v cpanm) --sudo --notest Test::Nginx IPC::Run Test2::Util > build.log 2>&1 || (cat build.log && exit 1) - wget -O - https://openresty.org/package/pubkey.gpg | sudo apt-key add - - echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/openresty.list - sudo apt-get update From b952dc1fc8344f5be0be1e282ba528a5310fb0c4 Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Tue, 10 Mar 2026 14:57:07 +0800 Subject: [PATCH 11/11] fix --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c80235a4ea..4d613c2e2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,7 +86,7 @@ install: #- if [ -n "$OPENSSL_VER" ] && [ ! -f download-cache/openssl-$OPENSSL_VER.tar.gz ]; then wget -P download-cache https://github.com/openssl/openssl/releases/download/openssl-$OPENSSL_VER/openssl-$OPENSSL_VER.tar.gz || wget -P download-cache https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz || wget -P download-cache https://www.openssl.org/source/old/${OPENSSL_VER//[a-z]/}/openssl-$OPENSSL_VER.tar.gz; fi - wget https://github.com/openresty/openresty-deps-prebuild/releases/download/v20230902/boringssl-20230902-x64-focal.tar.gz - wget https://github.com/openresty/openresty-deps-prebuild/releases/download/v20230902/curl-h3-x64-focal.tar.gz - - git clone --revision=efb42ee915b079b6b5e7fb24ee2afd7be44fa2bf --depth=1 https://github.com/openresty/test-nginx.git + - git clone https://github.com/openresty/test-nginx.git - git clone https://github.com/openresty/openresty.git ../openresty - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx - git clone https://github.com/openresty/openresty-devel-utils.git