UM_debug_script.sh

#!/usr/bin/env bash
#
# UM_debug_script.sh — diagnostic collector for a live Ultra Messaging process.
#
############################################################################
#  Copyright (c) 2015-2026 Informatica Corporation  Permission is granted to
#  licensees to use or alter this software for any purpose, including
#  commercial applications, according to the terms laid out in the Software
#  License Agreement.
#
#  This source code example is provided by Informatica for educational and
#  evaluation purposes only.
#
#  THE SOFTWARE IS PROVIDED "AS IS" AND INFORMATICA DISCLAIMS ALL WARRANTIES
#  EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION, ANY IMPLIED WARRANTIES
#  OF NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
#  INFORMATICA DOES NOT WARRANT THAT USE OF THE SOFTWARE WILL BE
#  UNINTERRUPTED OR ERROR-FREE.  INFORMATICA SHALL NOT, UNDER ANY
#  CIRCUMSTANCES, BE LIABLE TO LICENSEE FOR LOST PROFITS, CONSEQUENTIAL,
#  INCIDENTAL, SPECIAL OR INDIRECT DAMAGES ARISING OUT OF OR RELATED TO
#  THIS AGREEMENT OR THE TRANSACTIONS CONTEMPLATED HEREUNDER, EVEN IF
#  INFORMATICA HAS BEEN APPRISED OF THE LIKELIHOOD OF SUCH DAMAGES.
#
#  CAUTION:
#  --------
#  Exercise caution running the script in a production environment.
#  Perform a test run in UAT to determine if the generated debug log is not
#  excessively large and that there are no errors. Also check that
#  permissions are granted to run GDB on the process.
############################################################################

set -o pipefail
IFS=$'\n\t'

readonly VERSION='3.10'

# ---- defaults -------------------------------------------------------------

PATH=$PATH:/usr/sbin

UM_SLEEP_SEC=60
UM_BASE_DIR=/tmp/UM
: "${LBM_DEBUG_MASK:=0xffffffff0000DFB9}"
UM_CREATE_CORE_FILE="n"
UM_PID_APPNAME=""
UM_WEBMON_IPPORT=""

# ---- helpers --------------------------------------------------------------

log()  { printf '%s: %s\n' "$0" "$*"; }
warn() { printf '%s: WARNING: %s\n' "$0" "$*" >&2; }
die()  { printf '%s: ERROR: %s\n'   "$0" "$*" >&2; exit 1; }
have() { command -v "$1" >/dev/null 2>&1; }

declare -a OK_STEPS=()
declare -a SKIPPED_STEPS=()
declare -a FAILED_STEPS=()

# run_step "label" cmd args...
#   Invokes cmd; records OK or FAILED based on exit status. A failure logs a
#   warning but never aborts the script.
run_step() {
    local label=$1; shift
    if "$@"; then
        OK_STEPS+=("$label")
    else
        local rc=$?
        FAILED_STEPS+=("$label (exit $rc)")
        warn "$label failed (exit $rc)"
    fi
}

skip() {
    local label=$1; shift
    local reason=$*
    SKIPPED_STEPS+=("$label: $reason")
    log "skipping $label ($reason)"
}

# ---- trap / cleanup -------------------------------------------------------

DEBUG_MASK_SET=0
TCPDUMP_PID=""
TMPDIR_RUN=""
UM_DIR=""
pids=""

cleanup() {
    local rc=$?
    # Best-effort: each step tolerates the previous step's failure.

    if [[ -n "$TCPDUMP_PID" ]] && kill -0 "$TCPDUMP_PID" 2>/dev/null; then
        kill "$TCPDUMP_PID" 2>/dev/null || true
    fi

    if (( DEBUG_MASK_SET )) && have gdb && [[ -n "$pids" && -n "$UM_DIR" ]] \
       && [[ -r "$UM_DIR/debug_log_stop.gdb" ]]; then
        printf '%s: cleanup: restoring debug mask to 0 on PID %s\n' "$0" "$pids" >&2
        if gdb -x "$UM_DIR/debug_log_stop.gdb" -p "$pids" -batch \
               >/dev/null 2>&1; then
            DEBUG_MASK_SET=0
        else
            printf '%s: WARNING: cleanup: failed to restore debug mask on PID %s — check manually!\n' \
                "$0" "$pids" >&2
        fi
    fi

    if [[ -n "$TMPDIR_RUN" && -d "$TMPDIR_RUN" ]]; then
        rm -rf "$TMPDIR_RUN"
    fi

    exit "$rc"
}
trap cleanup EXIT INT TERM HUP

# ---- help -----------------------------------------------------------------

printhelp() {
    cat <<EOF
$0 ($VERSION)

Collects diagnostic evidence from a live Ultra Messaging process. Attempts:
netstat, pstack (x4), top, pidstat, lsof, tcpdump, webmon scrape, UM debug
log capture via gdb, and optionally gcore + libscollector packaging.

Usage: $0 -p <pid|appname> [-m mask] [-t secs] [-d dir] [-w ip:port] [-c]

  -c              Also generate a core file via gcore and package it via
                  libscollector.sh (which must live next to this script).
  -d <dir>        Output directory. Default: /tmp/UM/<pid>.
                  UM_DEBUGLOG_FILE env var overrides just the debug log path.
  -m <mask>       UM debug log mask. Default: \$LBM_DEBUG_MASK if set,
                  else 0xffffffff0000DFB9.
  -p <pid|app>    Required. Numeric values are interpreted as PIDs;
                  non-numeric values are resolved via pgrep/ps.
  -t <secs>       Duration to accumulate debug output. Default: 60.
  -w <ip:port>    Webmon endpoint to scrape via wget. Default for tnwgd and
                  umestored: 127.0.0.1:8456. Not used otherwise.

To exercise topic resolution, consider also running lbmrcv / lbmtrreq, e.g.:
  lbmtrreq -i 1000 -d 20 -L 20 -a -q -w -c <your_TR_config>

Output is archived to <dir>/UM_debug_script_output.<pid>.tar.gz for upload.
EOF
}

# ---- argument parsing -----------------------------------------------------

if [[ $# -lt 1 ]]; then
    printhelp
    exit 0
fi

log "version $VERSION"

while getopts ":t:d:cm:p:w:h" opt; do
    case "$opt" in
        t) UM_SLEEP_SEC=$OPTARG ;;
        c) UM_CREATE_CORE_FILE=y ;;
        d) UM_BASE_DIR=$OPTARG ;;
        m) LBM_DEBUG_MASK=$OPTARG ;;
        p) UM_PID_APPNAME=$OPTARG ;;
        w) UM_WEBMON_IPPORT=$OPTARG ;;
        h) printhelp; exit 0 ;;
        \?) printhelp; die "invalid option: -$OPTARG" ;;
        :)  printhelp; die "option -$OPTARG requires an argument" ;;
    esac
done

[[ -n "$UM_PID_APPNAME" ]] || { printhelp; die "-p is required"; }

# ---- resolve PID ----------------------------------------------------------

if [[ "$UM_PID_APPNAME" =~ ^[0-9]+$ ]]; then
    pids=$UM_PID_APPNAME
    log "checking PID $pids ..."
    if have ps; then
        app_chk=$(ps -p "$pids" -o comm= 2>/dev/null)
    else
        app_chk=""
    fi
    [[ -n "$app_chk" ]] || die "PID $pids not found"
    log "PID $pids belongs to: $app_chk"
else
    log "resolving application name <$UM_PID_APPNAME> ..."
    if have pgrep; then
        pids=$(pgrep -x -- "$UM_PID_APPNAME" 2>/dev/null || true)
    elif have pidof; then
        pids=$(pidof "$UM_PID_APPNAME" 2>/dev/null || true)
    else
        # last-ditch: scan ps output
        pids=$(ps -Ao pid,comm= 2>/dev/null \
               | awk -v app="$UM_PID_APPNAME" '$2==app {print $1}' \
               | tr '\n' ' ')
    fi

    [[ -n "$pids" ]] || die "no process found for application: $UM_PID_APPNAME"

    read -r -a pid_array <<< "$pids"
    if (( ${#pid_array[@]} > 1 )); then
        printf '%s: multiple processes match %s — pick one and re-run with -p <pid>:\n' \
            "$0" "$UM_PID_APPNAME" >&2
        ps -o pid,etime,comm,args -p "${pid_array[@]}" >&2 || true
        die "ambiguous -p: multiple PIDs"
    fi
    pids=${pid_array[0]}
    app_chk=$UM_PID_APPNAME
    log "PID: $pids  APP: $app_chk"
fi

# Final webmon default, applied *after* app detection so the logged value is
# the one actually used.
if [[ -z "$UM_WEBMON_IPPORT" ]]; then
    if [[ "$app_chk" == tnwgd || "$app_chk" == umestored ]]; then
        UM_WEBMON_IPPORT=127.0.0.1:8456
        log "$app_chk detected, using default webmon: $UM_WEBMON_IPPORT"
    fi
fi

log "LBM_DEBUG_MASK:     $LBM_DEBUG_MASK"
log "UM_SLEEP_SEC:       $UM_SLEEP_SEC"
log "UM_BASE_DIR:        $UM_BASE_DIR"
log "PID/APPNAME:        $UM_PID_APPNAME (resolved to $pids)"
log "UM_CREATE_CORE_FILE:$UM_CREATE_CORE_FILE"
log "UM_WEBMON_IPPORT:   ${UM_WEBMON_IPPORT:-<none>}"

# ---- output directory -----------------------------------------------------

mkdir -p "$UM_BASE_DIR" || die "could not create $UM_BASE_DIR"
# Absolutize so later `cd "$UM_BASE_DIR"` works regardless of cwd.
UM_BASE_DIR=$(cd "$UM_BASE_DIR" && pwd) || die "cannot resolve $UM_BASE_DIR"
UM_DIR="$UM_BASE_DIR/$pids"
mkdir -p "$UM_DIR" || die "could not create $UM_DIR"
[[ -d "$UM_DIR" ]] || die "$UM_DIR is not a directory"

# UM debug log file path: absolute, under UM_DIR, without relying on
# readlink -f (which differs between GNU and BSD coreutils).
: "${UM_DEBUGLOG_FILE:=$(cd "$UM_DIR" && pwd)/UMdbglog.txt}"

cd "$UM_DIR" || die "cannot cd into $UM_DIR"

TMPDIR_RUN=$(mktemp -d -t um_debug.XXXXXX) \
    || die "could not create temp directory"

# ---- gdb command files ----------------------------------------------------

# Note: `\$` escapes prevent the shell from expanding gdb internal vars.
cat > UM_util.gdb <<End-of-message
# UM gdb helpers

define UM_set_debug_filename
  set \$debug_file_name = (\$arg0)
  call lbm_debug_filename(\$debug_file_name)
  call lbm_debug_noflush(1)
end

define UM_set_debug_mask
  set \$debug_mask_value = (\$arg0)
  call (void)mul_set_dmask(\$debug_mask_value)
end
End-of-message

cp UM_util.gdb debug_log_filename.gdb
cp UM_util.gdb debug_log_start.gdb
cp UM_util.gdb debug_log_stop.gdb

log "UM debug log file: $UM_DEBUGLOG_FILE"

cat >> debug_log_filename.gdb <<End-of-message
UM_set_debug_filename "$UM_DEBUGLOG_FILE"
quit
End-of-message

cat >> debug_log_start.gdb <<End-of-message
UM_set_debug_mask $LBM_DEBUG_MASK
quit
End-of-message

cat >> debug_log_stop.gdb <<End-of-message
UM_set_debug_mask 0
quit
End-of-message

# ---- collectors -----------------------------------------------------------

collect_uname() { uname -a > uname.log; }

collect_tcpdump_start() {
    # -i lo -i any requires recent libpcap; older versions silently ignore
    # one of the interfaces. Errors go to tcpdump.err for post-mortem.
    tcpdump -i lo -i any -s0 -v -G 500 -C 50 \
        -w "$UM_DIR/${pids}.pcap" 2>tcpdump.err &
    TCPDUMP_PID=$!
}

collect_tcpdump_stop() {
    if [[ -n "$TCPDUMP_PID" ]] && kill -0 "$TCPDUMP_PID" 2>/dev/null; then
        kill "$TCPDUMP_PID" 2>/dev/null || true
    fi
    TCPDUMP_PID=""
}

collect_netstat() {
    if have ss; then
        {
            ss -nap
            echo "----"
            ss -s
        } > netstat.log
    elif have netstat; then
        {
            netstat -nap
            netstat -s
            netstat -g
        } > netstat.log
    else
        return 1
    fi
}

collect_top_1()     { top -H -n 1 -b -p "$pids" > top1.log; }
collect_top_2()     { top -H -n 1 -b -p "$pids" > top2.log; }
collect_free()      { free -m > free.log; }
collect_cpuinfo()   { [[ -r /proc/cpuinfo ]] && cat /proc/cpuinfo > cpuinfo.log; }
collect_meminfo()   { [[ -r /proc/meminfo ]] && cat /proc/meminfo > meminfo.log; }
collect_pidstat()   { pidstat -p "$pids" -drtuw > pidstat.log; }
collect_lsof()      { lsof -p "$pids" > lsof.log; }

collect_pstacks() {
    local i
    for i in 1 2 3 4; do
        pstack "$pids" > "pstack.$i.log" || return 1
        (( i < 4 )) && sleep "$i"
    done
}

collect_pstack_last() { pstack "$pids" > pstack.last.log; }

collect_webmon() {
    local url=$UM_WEBMON_IPPORT

    # Honor a caller-supplied scheme; default to http.
    if [[ "$url" != http://* && "$url" != https://* ]]; then
        url="http://$url"
    fi

    # Warn if the port isn't actually listening in the target process.
    if [[ -r lsof.log ]]; then
        local port
        port=${UM_WEBMON_IPPORT##*:}
        port=${port%%/*}
        if [[ "$port" =~ ^[0-9]+$ ]] && ! grep -q ":$port" lsof.log; then
            warn "webmon port $port not found in lsof.log — listening TCP ports:"
            grep -i 'TCP' lsof.log | grep -i 'LISTEN' >&2 || true
        fi
    fi

    log "running wget for webmon capture: $url"
    wget -rk -E --timeout=10 --tries=2 "$url" -o wget.log -P webmondir
}

# ---- begin collection -----------------------------------------------------

date > date.1.log

run_step "uname" collect_uname

if have tcpdump; then
    run_step "tcpdump-start" collect_tcpdump_start
else
    skip "tcpdump" "not installed"
fi

if have ss || have netstat; then
    run_step "netstat" collect_netstat
else
    skip "netstat" "neither ss nor netstat installed"
fi

if have top;     then run_step "top-1"    collect_top_1;    else skip "top"     "not installed"; fi
if have free;    then run_step "free"     collect_free;     else skip "free"    "not installed"; fi

run_step "cpuinfo" collect_cpuinfo
run_step "meminfo" collect_meminfo

if have pidstat; then run_step "pidstat"  collect_pidstat;  else skip "pidstat" "not installed"; fi
if have lsof;    then run_step "lsof"     collect_lsof;     else skip "lsof"    "not installed"; fi

if have pstack; then
    log "running pstacks (x4)"
    run_step "pstacks" collect_pstacks
else
    skip "pstack" "not installed"
fi

# Webmon
if [[ -n "$UM_WEBMON_IPPORT" ]]; then
    if have wget; then
        run_step "webmon" collect_webmon
    else
        skip "webmon" "wget not installed"
    fi
fi

# Confirm liblbm loaded (non-fatal)
if [[ -r lsof.log ]] && ! grep -q liblbm lsof.log; then
    warn "could not verify liblbm is loaded by $app_chk"
fi

# ---- gdb-driven debug-log capture -----------------------------------------

if ! have gdb; then
    skip "gdb-debug-log" "gdb not installed"
else
    log "setting UM debug log filename via gdb"
    if ! gdb -x debug_log_filename.gdb -p "$pids" -batch 2>gdb.filename.err; then
        warn "gdb debug_log_filename failed — debug log capture will be skipped"
        warn "  (common cause: ptrace_scope=1, or process owned by another UID)"
    else
        log "starting UM debug log capture"
        if ! gdb -x debug_log_start.gdb -p "$pids" -batch 2>gdb.start.err; then
            warn "gdb debug_log_start failed — no debug output will be collected"
        else
            DEBUG_MASK_SET=1
            OK_STEPS+=("gdb-debug-log-start")
            log "sleeping ${UM_SLEEP_SEC}s to accumulate debug output"
            sleep "$UM_SLEEP_SEC"
            log "stopping UM debug log capture"
            if gdb -x debug_log_stop.gdb -p "$pids" -batch 2>gdb.stop.err; then
                DEBUG_MASK_SET=0
                OK_STEPS+=("gdb-debug-log-stop")
            else
                warn "gdb debug_log_stop failed — mask may still be set; trap will retry"
            fi
        fi
    fi
fi

# ---- post-capture snapshots -----------------------------------------------

date > date.2.log
if have pstack; then run_step "pstack-last" collect_pstack_last; fi
if have top;    then run_step "top-2"       collect_top_2;       fi

# Stop tcpdump and move any pcap files (rotation creates .pcap0, .pcap1, ...)
if [[ -n "$TCPDUMP_PID" ]]; then
    collect_tcpdump_stop
    shopt -s nullglob
    for pcap in "$UM_DIR/${pids}".pcap*; do
        # Files are already in UM_DIR; nothing to move. Glob here exists just
        # to note them in logs.
        log "captured: $pcap"
    done
    shopt -u nullglob
fi

# ---- core + libscollector -------------------------------------------------

if [[ "$UM_CREATE_CORE_FILE" != "y" ]]; then
    log "no core file requested"
elif ! have gcore; then
    skip "gcore" "not installed"
else
    log "generating core via gcore"
    COREFILE="core.$pids"
    # gcore writes to cwd ($UM_DIR) as core.$pid unless overridden.
    if ! gcore "$pids" >gcore.log 2>&1; then
        warn "gcore failed (see gcore.log)"
    elif [[ ! -r "$COREFILE" ]]; then
        warn "gcore reported success but $UM_DIR/$COREFILE not found — core may have been captured by systemd-coredump or similar; check /proc/sys/kernel/core_pattern"
    else
        OK_STEPS+=("gcore")

        # Resolve the executable path via /proc. Linux-only; on other
        # platforms libscollector's own checks will sort it out.
        EXECFILE=""
        if [[ -r "/proc/$pids/exe" ]]; then
            EXECFILE=$(readlink "/proc/$pids/exe" 2>/dev/null || true)
        fi

        if [[ -z "$EXECFILE" ]]; then
            warn "could not resolve executable for PID $pids — skipping libscollector"
        else
            # Locate libscollector.sh alongside this script (readlink -f
            # behavior varies; use a portable-enough alternative).
            script_dir=$(cd "$(dirname "$0")" && pwd)
            libscollector="${LIBSCOLLECTOR:-$script_dir/libscollector.sh}"

            if [[ ! -x "$libscollector" ]]; then
                warn "libscollector.sh not found or not executable: $libscollector — skipping"
            else
                log "invoking $libscollector"
                if "$libscollector" "$EXECFILE" "$COREFILE"; then
                    OK_STEPS+=("libscollector")
                    # libscollector produced COREFILE.tar.gz (or .Z).
                    # Remove the bare core since it's now inside the archive.
                    if compgen -G "${COREFILE}_libs_all.tar.*" >/dev/null; then
                        rm -f "$COREFILE"
                    fi
                else
                    warn "libscollector returned non-zero"
                fi
            fi
        fi
    fi
fi

# ---- summary + archive ----------------------------------------------------

print_summary() {
    # Use a local IFS so array expansion joins with spaces, not newlines.
    local IFS=' '
    printf '%s: ========== collection summary ==========\n' "$0"
    if (( ${#OK_STEPS[@]} )); then
        printf '%s: OK:      %s\n' "$0" "${OK_STEPS[*]}"
    fi
    local s f
    for s in "${SKIPPED_STEPS[@]}"; do
        printf '%s: SKIPPED: %s\n' "$0" "$s"
    done
    for f in "${FAILED_STEPS[@]}"; do
        printf '%s: FAILED:  %s\n' "$0" "$f"
    done
    printf '%s: ========================================\n' "$0"
}

print_summary

if have tar; then
    mytar="UM_debug_script_output.${pids}.tar.gz"
    log "archiving $UM_DIR"
    (cd "$UM_BASE_DIR" && tar -czf "$mytar" "$pids")
    log "please upload $UM_BASE_DIR/$mytar"
else
    warn "tar not installed — upload the contents of $UM_DIR instead"
fi

log "SCRIPT DONE"
exit 0