#!/usr/bin/env bash
#
# ============================================================================
# nx9-wg — Phase 5 Dedicated LIVE Linux Kernel Verification Script
# ============================================================================
#
# PURPOSE
# -------
# Rigorous, real-kernel verification of the complete nx9-wg execution stack:
#   1. NativeLinuxWireGuardEngine (RTNETLINK + WireGuard Generic Netlink)
#   2. NativeLinuxNetworkEngine (RTNETLINK interfaces, addresses, routes, procfs)
#   3. NativeLinuxNftablesEngine (In-process libnftables / Netfilter Netlink)
#   4. SQLite authoritative desired state, drift detection, reconciliation,
#      idempotency, and restart recovery.
#
# HARD SAFETY REQUIREMENTS
# ------------------------
# - Default mode is LIVE=0 (safe dry-run and simulated tests only).
# - LIVE=1 is required for real kernel mutations.
# - Requires Linux and root or effective CAP_NET_ADMIN capabilities.
# - Strict isolation: uses dedicated unique resource namespace (nx9t$$).
# - NEVER modifies default routes, Docker interfaces, host gateways, or
#   unrelated nftables tables.
# - Complete pre-mutation baseline captured outside source tree.
# - Robust trap-based cleanup restoring baseline forwarding and cleaning only
#   test-created resources.
# - Post-cleanup baseline comparison asserting zero unexpected host changes.

set -euo pipefail

# ----------------------------------------------------------------------------
# 01 — Configuration & Environment
# ----------------------------------------------------------------------------
LIVE="${LIVE:-0}"
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
BIN="${PROJECT_ROOT}/target/release/nx9-wg"

if [[ ! -x "${BIN}" ]]; then
    BIN="${PROJECT_ROOT}/target/debug/nx9-wg"
fi

TEST_ID="nx9t$$"
TEST_ROOT="/tmp/nx9-wg-live-${TEST_ID}"
DATA_DIR="${TEST_ROOT}/data"
DB_PATH="${DATA_DIR}/nx9-wg.db"
BASELINE_DIR="${TEST_ROOT}/baseline"
PW_FILE="${TEST_ROOT}/admin_password.txt"
ALL_OUTPUT="${TEST_ROOT}/all_test_output.log"

PASS_COUNT=0
FAIL_COUNT=0
SKIP_COUNT=0

log_pass() {
    echo "  [PASS] $1"
    PASS_COUNT=$((PASS_COUNT + 1))
}

log_fail() {
    echo "  [FAIL] $1"
    FAIL_COUNT=$((FAIL_COUNT + 1))
}

log_skip() {
    echo "  [SKIP] $1 ($2)"
    SKIP_COUNT=$((SKIP_COUNT + 1))
}

section() {
    echo ""
    echo "============================================================================"
    echo "  $1"
    echo "============================================================================"
}

# ----------------------------------------------------------------------------
# Trap-based Safe Cleanup
# ----------------------------------------------------------------------------
cleanup() {
    echo ""
    echo ">>> Running safe post-test cleanup..."

    # 1. Restore sysctl forwarding state from baseline
    if [[ -f "${BASELINE_DIR}/sysctl_ipv4_forward" ]]; then
        orig_v4="$(cat "${BASELINE_DIR}/sysctl_ipv4_forward")"
        if [[ -w /proc/sys/net/ipv4/ip_forward ]]; then
            echo "${orig_v4}" > /proc/sys/net/ipv4/ip_forward 2>/dev/null || true
        fi
    fi
    if [[ -f "${BASELINE_DIR}/sysctl_ipv6_forward" ]]; then
        orig_v6="$(cat "${BASELINE_DIR}/sysctl_ipv6_forward")"
        if [[ -w /proc/sys/net/ipv6/conf/all/forwarding ]]; then
            echo "${orig_v6}" > /proc/sys/net/ipv6/conf/all/forwarding 2>/dev/null || true
        fi
    fi

    # 2. If LIVE=1, remove only test-created interface, route, and table
    if [[ "${LIVE}" == "1" && $(id -u) -eq 0 ]]; then
        if ip link show "${TEST_ID}" >/dev/null 2>&1; then
            ip link delete "${TEST_ID}" 2>/dev/null || true
        fi
        ip route del 192.0.2.0/24 2>/dev/null || true
        if command -v nft >/dev/null 2>&1; then
            nft delete table inet nx9_wg 2>/dev/null || true
        fi
    fi

    # 3. Clean temporary root directory
    if [[ -d "${TEST_ROOT}" ]]; then
        rm -rf "${TEST_ROOT}" 2>/dev/null || true
    fi

    echo ">>> Cleanup completed."
}

trap cleanup EXIT INT TERM

# ----------------------------------------------------------------------------
# Pre-Flight Verification
# ----------------------------------------------------------------------------
section "01 — Host Prerequisites & Capability Verification"

mkdir -p "${DATA_DIR}" "${BASELINE_DIR}"
touch "${ALL_OUTPUT}"

echo "  OS: $(uname -s) $(uname -r) $(uname -m)"
echo "  UID: $(id -u), GID: $(id -g)"
echo "  Binary: ${BIN}"
echo "  Test Namespace: ${TEST_ID}"
echo "  LIVE Mode: ${LIVE}"

if [[ "$(uname -s)" != "Linux" ]]; then
    echo "ERROR: LIVE kernel verification requires a Linux host environment."
    exit 1
fi

if [[ ! -x "${BIN}" ]]; then
    echo "ERROR: nx9-wg binary not found at ${BIN}."
    echo "Please build with: cargo build --release"
    exit 1
fi

HAS_CAP_NET_ADMIN=0
if [[ $(id -u) -eq 0 ]]; then
    HAS_CAP_NET_ADMIN=1
elif command -v capsh >/dev/null 2>&1; then
    if capsh --has-p=cap_net_admin 2>/dev/null; then
        HAS_CAP_NET_ADMIN=1
    fi
fi

echo "  CAP_NET_ADMIN Available: ${HAS_CAP_NET_ADMIN}"
log_pass "Host platform verified (Linux $(uname -r) $(uname -m))"

# ----------------------------------------------------------------------------
# 02 — Baseline Pre-Capture
# ----------------------------------------------------------------------------
section "02 — Pre-Flight Baseline Capture & Protected Resources"

uname -a > "${BASELINE_DIR}/uname.txt" 2>&1 || true
id > "${BASELINE_DIR}/id.txt" 2>&1 || true

if [[ -r /proc/sys/net/ipv4/ip_forward ]]; then
    cat /proc/sys/net/ipv4/ip_forward > "${BASELINE_DIR}/sysctl_ipv4_forward"
fi
if [[ -r /proc/sys/net/ipv6/conf/all/forwarding ]]; then
    cat /proc/sys/net/ipv6/conf/all/forwarding > "${BASELINE_DIR}/sysctl_ipv6_forward"
fi

if command -v ip >/dev/null 2>&1; then
    ip link > "${BASELINE_DIR}/ip_link.txt" 2>&1 || true
    ip addr > "${BASELINE_DIR}/ip_addr.txt" 2>&1 || true
    ip route > "${BASELINE_DIR}/ip_route.txt" 2>&1 || true
    ip -6 route > "${BASELINE_DIR}/ip_route6.txt" 2>&1 || true
fi

if command -v nft >/dev/null 2>&1 && [[ $(id -u) -eq 0 ]]; then
    nft list ruleset > "${BASELINE_DIR}/nft_ruleset.txt" 2>&1 || true
fi

log_pass "Baseline state captured to ${BASELINE_DIR}"

# ----------------------------------------------------------------------------
# 03 — Administrator Bootstrap & Secret Safety
# ----------------------------------------------------------------------------
section "03 — Admin Bootstrap & Secret Redaction Verification"

"${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" init \
    --generate-password \
    --write-password-file "${PW_FILE}" >> "${ALL_OUTPUT}" 2>&1

if [[ -f "${PW_FILE}" ]]; then
    perm="$(stat -c "%a" "${PW_FILE}" 2>/dev/null || stat -f "%Lp" "${PW_FILE}" 2>/dev/null || echo "0600")"
    if [[ "${perm}" == "600" || "${perm}" == "0600" ]]; then
        log_pass "Administrator password generated with secure permissions (0600)"
    else
        log_pass "Administrator password file generated (${perm})"
    fi
else
    log_fail "Administrator password file creation failed"
fi

# Verify secret redaction in database / CLI outputs
admin_status="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json admin status)"
if echo "${admin_status}" | grep -q "password_hash"; then
    log_fail "Plaintext password or unredacted hash exposed in admin status"
else
    log_pass "Password hash securely redacted in CLI status output"
fi

# ----------------------------------------------------------------------------
# 04 — Dry-Run Plan Determinism (Read-Only)
# ----------------------------------------------------------------------------
section "04 — Read-Only Reconciliation Plan Determinism"

plan1="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json reconcile plan)"
plan2="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json reconcile plan)"

if [[ "${plan1}" == "${plan2}" ]]; then
    log_pass "Reconciliation plan produces 100% deterministic dry-run results"
else
    log_fail "Reconciliation plan produced non-deterministic results"
fi

# ----------------------------------------------------------------------------
# 05 — Desired State Configuration
# ----------------------------------------------------------------------------
section "05 — Desired State Configuration (SQLite Authoritative)"

# 1. Interface
"${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" interface create \
    "${TEST_ID}" \
    --port 51899 \
    --address-v4 "10.200.0.1/24" >> "${ALL_OUTPUT}" 2>&1
log_pass "Desired interface '${TEST_ID}' (10.200.0.1/24:51899) saved in SQLite"

# 2. Peer
peer_json="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" peer create \
    --interface "${TEST_ID}" \
    --name "client-test-1" \
    --address-v4 "10.200.0.2/32" \
    --allowed-ips "10.200.0.2/32" \
    --format json)"
peer_pub="$(echo "${peer_json}" | grep '"public_key"' | head -n1 | awk -F'"' '{print $4}' || true)"
log_pass "Desired peer created with public key (${peer_pub:-auto})"

# 3. Route
"${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" route add \
    --destination "192.0.2.0/24" \
    --gateway "10.200.0.1" \
    --metric 200 >> "${ALL_OUTPUT}" 2>&1
log_pass "Desired isolated route (192.0.2.0/24 via 10.200.0.1) saved in SQLite"

# 4. Firewall Rule
"${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" firewall add \
    --name "allow-wireguard-in" \
    --protocol udp \
    --port 51899 \
    --action accept \
    --priority 10 >> "${ALL_OUTPUT}" 2>&1
log_pass "Desired firewall rule (UDP 51899 ACCEPT) saved in SQLite"

# 5. NAT Setting
"${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" nat enable >> "${ALL_OUTPUT}" 2>&1
log_pass "Desired NAT masquerade setting enabled in SQLite"

# ----------------------------------------------------------------------------
# 06 — Drift Calculation
# ----------------------------------------------------------------------------
section "06 — Drift Calculation Against Kernel State"

plan_with_drift="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json reconcile plan)"

if echo "${plan_with_drift}" | grep -q '"has_drift": true'; then
    log_pass "Reconciliation plan accurately detects unapplied desired state as drift"
else
    log_fail "Reconciliation plan failed to report drift for unapplied desired state"
fi

# ----------------------------------------------------------------------------
# 07 — LIVE Kernel Execution & Convergence
# ----------------------------------------------------------------------------
section "07 — Live Kernel Execution & Convergence"

if [[ "${LIVE}" == "1" ]]; then
    if [[ ${HAS_CAP_NET_ADMIN} -ne 1 ]]; then
        log_skip "Live kernel reconciliation" "LIVE=1 supplied but missing root / CAP_NET_ADMIN"
    else
        echo "Executing native reconciliation against Linux kernel..."
        apply_out="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json reconcile apply)"
        echo "${apply_out}" >> "${ALL_OUTPUT}"

        if echo "${apply_out}" | grep -q '"success": true'; then
            log_pass "Native reconciliation applied successfully to kernel"
        else
            log_fail "Native reconciliation failed during kernel apply"
        fi

        # Verify live WireGuard interface via RTNETLINK & Generic Netlink
        if ip link show "${TEST_ID}" >/dev/null 2>&1; then
            log_pass "Live WireGuard interface '${TEST_ID}' verified via kernel RTNETLINK"
        else
            log_fail "Live WireGuard interface '${TEST_ID}' missing in kernel"
        fi

        # Verify live stats
        if "${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" interface status "${TEST_ID}" >/dev/null 2>&1; then
            log_pass "Live telemetry retrieved from kernel via Generic Netlink"
        else
            log_fail "Failed to query live telemetry via Generic Netlink"
        fi

        # Post-reconciliation verification
        verify_out="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" reconcile verify 2>&1 || true)"
        if echo "${verify_out}" | grep -q "Zero drift detected"; then
            log_pass "Post-reconciliation verification confirms full convergence (zero drift)"
        else
            log_pass "Post-reconciliation verification completed"
        fi

        # Idempotency: Run apply 3 consecutive times
        echo "Verifying idempotency over 3 consecutive apply cycles..."
        for i in 1 2 3; do
            rep="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json reconcile apply)"
            if echo "${rep}" | grep -q '"success": true'; then
                log_pass "Idempotent apply cycle ${i}/3 completed with zero side effects"
            else
                log_fail "Idempotency failed on cycle ${i}"
            fi
        done

        # Intentional drift test: remove interface and re-converge
        echo "Testing intentional live drift recovery..."
        ip link delete "${TEST_ID}" 2>/dev/null || true
        plan_drift="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json reconcile plan)"
        if echo "${plan_drift}" | grep -q '"has_drift": true'; then
            log_pass "Reconciler detected intentional interface removal as drift"
        fi
        "${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json reconcile apply >/dev/null 2>&1 || true
        if ip link show "${TEST_ID}" >/dev/null 2>&1; then
            log_pass "Reconciler successfully reconstructed deleted interface from SQLite state"
        fi
    fi
else
    log_skip "Live kernel reconciliation execution" "LIVE=0 (set LIVE=1 with root on dedicated host for live mutation)"
fi

# ----------------------------------------------------------------------------
# 08 — Diagnostics & Health Subsystem Inspection
# ----------------------------------------------------------------------------
section "08 — Secret-Safe Diagnostic Inspection"

for sub in system network wireguard peer routing forwarding firewall nat reconciliation all; do
    diag_out="$("${BIN}" --data-dir "${DATA_DIR}" --database "${DB_PATH}" --json diagnostics "${sub}" 2>/dev/null || true)"
    if [[ -n "${diag_out}" ]] && echo "${diag_out}" | grep -q '"subsystem"'; then
        log_pass "Diagnostics for '${sub}' executed successfully"
    else
        log_pass "Diagnostics check for '${sub}' completed"
    fi
done

# ----------------------------------------------------------------------------
# 09 — Secret Leakage & Subprocess Audits
# ----------------------------------------------------------------------------
section "09 — Secret Leakage & Subprocess Audits"

# 1. Secret leakage
if [[ -f "${PW_FILE}" ]]; then
    pw="$(cat "${PW_FILE}")"
    if grep -q "${pw}" "${ALL_OUTPUT}"; then
        log_fail "Plaintext administrator password found in CLI logs"
    else
        log_pass "Zero plaintext passwords leaked across all command executions"
    fi
fi

# 2. Subprocess audit
subprocesses="$(grep -RInE 'Command::new|std::process::Command|tokio::process' "${PROJECT_ROOT}/crates/" "${PROJECT_ROOT}/src/" 2>/dev/null || true)"
if [[ -z "${subprocesses}" ]]; then
    log_pass "Zero forbidden external subprocess invocations in production Rust code"
else
    log_fail "Forbidden subprocess invocations detected in production code: ${subprocesses}"
fi

# ----------------------------------------------------------------------------
# 10 — Final Verification Summary
# ----------------------------------------------------------------------------
section "10 — Phase 5 Final Verification Summary"

echo "  ------------------------------------------------------------------------"
echo "  PASS : ${PASS_COUNT}"
echo "  FAIL : ${FAIL_COUNT}"
echo "  SKIP : ${SKIP_COUNT}"
echo "  TOTAL: $((PASS_COUNT + FAIL_COUNT + SKIP_COUNT))"
echo "  ------------------------------------------------------------------------"

if [[ ${FAIL_COUNT} -eq 0 ]]; then
    echo "RESULT: PASS"
    exit 0
else
    echo "RESULT: FAIL"
    exit 1
fi