From bc82d48124c26c8192eb8bbb53abf35839a390c0 Mon Sep 17 00:00:00 2001 From: sawka Date: Mon, 27 Jan 2025 18:00:05 -0800 Subject: [PATCH] send SIGUSR1 to dump all goroutines (for debugging) --- cmd/server/main-server.go | 18 ++++++++++++++++++ pkg/util/utilfn/utilfn.go | 7 +++++++ 2 files changed, 25 insertions(+) diff --git a/cmd/server/main-server.go b/cmd/server/main-server.go index 06ce5639d8..4cfe5c76b4 100644 --- a/cmd/server/main-server.go +++ b/cmd/server/main-server.go @@ -25,6 +25,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/service" "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/util/shellutil" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wcloud" @@ -75,6 +76,9 @@ func installShutdownSignalHandlers() { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) go func() { + defer func() { + panichandler.PanicHandler("installShutdownSignalHandlers", recover()) + }() for sig := range sigCh { doShutdown(fmt.Sprintf("got signal %v", sig)) break @@ -82,6 +86,19 @@ func installShutdownSignalHandlers() { }() } +func installSIGUSR1Handler() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGUSR1) + go func() { + defer func() { + panichandler.PanicHandler("installSIGUSR1Handler", recover()) + }() + for range sigCh { + utilfn.DumpGoRoutineStacks() + } + }() +} + // watch stdin, kill server if stdin is closed func stdinReadWatch() { buf := make([]byte, 1024) @@ -296,6 +313,7 @@ func main() { createMainWshClient() installShutdownSignalHandlers() + installSIGUSR1Handler() startupActivityUpdate() go stdinReadWatch() go telemetryLoop() diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index 36cd4be169..fb04dd61c8 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -20,6 +20,7 @@ import ( "os/exec" "reflect" "regexp" + "runtime" "sort" "strconv" "strings" @@ -941,3 +942,9 @@ func HasBinaryData(data []byte) bool { } return false } + +func DumpGoRoutineStacks() { + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + os.Stdout.Write(buf[:n]) +}