nx9-wg/scripts/diagnose.sh

#!/usr/bin/env bash
# ==============================================================================
# nx9-wg Production Diagnostic Collector
# ==============================================================================
# PURPOSE:
#   Collects a comprehensive, non-destructive diagnostic snapshot across both
#   desired SQLite state and live Linux kernel networking state:
#     - Systemd service & journal log health
#     - UDP socket bindings & conflict inspection
#     - Kernel IP link, address, and routing status
#     - Kernel WireGuard link/interface and peer telemetry (via secret-safe 'wg show')
#     - Netfilter / nftables ruleset in 'table inet nx9_wg'
#     - Linux sysctl IP packet forwarding
#     - Application-level diagnostic subsystem inspections
#
# PREREQUISITES:
#   - Root privileges (recommended for kernel/socket inspection, or run via sudo)
#
# SECURITY INVARIANTS:
#   - NEVER calls 'wg showconf' (which prints private keys in cleartext).
#   - Relies exclusively on 'wg show' which masks private keys.
#   - Strictly non-destructive: only performs read-only inspections.
#
# USAGE:
#   sudo bash scripts/diagnose.sh
#   ./scripts/diagnose.sh
# ==============================================================================

set -uo pipefail

BIN="/usr/local/bin/nx9-wg"
if [[ ! -x "${BIN}" ]]; then
  SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
  if [[ -x "${ROOT_DIR}/target/release/nx9-wg" ]]; then
    BIN="${ROOT_DIR}/target/release/nx9-wg"
  elif [[ -x "${ROOT_DIR}/target/debug/nx9-wg" ]]; then
    BIN="${ROOT_DIR}/target/debug/nx9-wg"
  fi
fi

section() {
  echo -e "\n================================================================="
  echo -e "\033[1;36m>> $*\033[0m"
  echo -e "================================================================="
}

subsection() {
  echo -e "\n\033[1;33m--- $*\033[0m"
}

section "1. System & Host Runtime Environment"
echo "Timestamp:    $(date --iso-8601=seconds)"
echo "Hostname:     $(hostname)"
echo "Kernel:       $(uname -r)"
echo "Architecture: $(uname -m)"
echo "Uptime:       $(uptime -p 2>/dev/null || uptime)"
if [[ -x "${BIN}" ]]; then
  echo "nx9-wg:       $("${BIN}" version | head -n 1)"
else
  echo "nx9-wg:       Binary not found"
fi

section "2. Systemd Service & Process State"
if command -v systemctl >/dev/null 2>&1; then
  subsection "Service Status (nx9-wg.service)"
  systemctl status nx9-wg --no-pager -l || true

  subsection "Recent Journalctl Logs (Last 30 entries)"
  journalctl -u nx9-wg -n 30 --no-pager || true
else
  echo "systemctl not available on this host."
fi

section "3. UDP Sockets & Listen Port Inspection"
if command -v ss >/dev/null 2>&1; then
  subsection "Active UDP Listen Sockets (ss -lunp)"
  ss -lunp 2>/dev/null || true
else
  echo "ss utility not found."
fi

section "4. Linux Network Interfaces & Addresses"
if command -v ip >/dev/null 2>&1; then
  subsection "Brief Interface State (ip -br link)"
  ip -br link show || true

  subsection "Brief IPv4 / IPv6 Addresses (ip -br addr)"
  ip -br addr show || true

  subsection "WireGuard Interface Addresses"
  if command -v wg >/dev/null 2>&1; then
    WG_INTERFACES="$(wg show interfaces 2>/dev/null || true)"
    if [[ -n "${WG_INTERFACES}" ]]; then
      for WG_IFACE in ${WG_INTERFACES}; do
        echo "Interface: ${WG_IFACE}"
        ip addr show dev "${WG_IFACE}" 2>/dev/null || true
      done
    else
      echo "No WireGuard interfaces reported by the kernel."
    fi
  else
    echo "wg utility not found; WireGuard interface-specific address inspection skipped."
  fi
else
  echo "ip utility not found."
fi

section "5. Kernel Routing Table"
if command -v ip >/dev/null 2>&1; then
  subsection "IPv4 Routes (ip route show)"
  ip route show || true

  subsection "IPv6 Routes (ip -6 route show)"
  ip -6 route show || true
fi

section "6. Kernel WireGuard Telemetry (Secret-Safe 'wg show')"
if command -v wg >/dev/null 2>&1; then
  wg show 2>&1 || echo "wg show returned non-zero (may require root privileges)."
else
  echo "wg utility not found on host."
fi

section "7. Netfilter / nftables Firewall State (table inet nx9_wg)"
if command -v nft >/dev/null 2>&1; then
  nft list table inet nx9_wg 2>/dev/null || echo "nftables table 'inet nx9_wg' not present."
else
  echo "nft utility not found on host."
fi

section "8. IP Packet Forwarding (Kernel Sysctl)"
echo -n "net.ipv4.ip_forward: "
cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo "Unable to read /proc/sys/net/ipv4/ip_forward"
echo -n "net.ipv6.conf.all.forwarding: "
cat /proc/sys/net/ipv6/conf/all/forwarding 2>/dev/null || echo "Unable to read /proc/sys/net/ipv6/conf/all/forwarding"

section "9. Application Desired State & Health Checks"
if [[ -x "${BIN}" ]]; then
  subsection "Appliance Health Check"
  "${BIN}" system health 2>&1 || true

  subsection "All Subsystems Diagnostics"
  "${BIN}" diagnostics all 2>&1 || true

  subsection "Reconciliation Drift Status"
  "${BIN}" reconcile status 2>&1 || true

  subsection "Live WireGuard Interface Status"
  "${BIN}" live interface list 2>&1 || true
else
  echo "nx9-wg binary not executable; skipping application-level diagnostics."
fi

section "Diagnostic Collection Complete"