feat: complete nx9-wg v0.8.0 platform

crates/nx9-wg-api/src/reconciliation.rs

@@ -32,10 +32,26 @@ pub struct ReconciliationPlan {
     pub forwarding_changes: usize,
 }
 
+/// Detailed status of reconciliation execution lifecycle.
+#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ReconciliationStatus {
+    #[default]
+    Plan,
+    Applying,
+    PartialFailure,
+    Failed,
+    Verifying,
+    Converged,
+    DriftRemains,
+}
+
 /// Final report of an executed reconciliation cycle.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ReconciliationReport {
     pub success: bool,
+    #[serde(default)]
+    pub status: ReconciliationStatus,
     pub executed_actions: usize,
     pub details: Vec<String>,
 }
@@ -45,6 +61,7 @@ pub struct ReconciliationEngine {
     state: AppState,
     wg_engine: Arc<dyn WireGuardEngine>,
     net_engine: Arc<dyn NetworkEngine>,
+    lock: Arc<tokio::sync::Mutex<()>>,
 }
 
 impl ReconciliationEngine {
@@ -58,6 +75,7 @@ impl ReconciliationEngine {
             state,
             wg_engine,
             net_engine,
+            lock: Arc::new(tokio::sync::Mutex::new(())),
         }
     }
 
@@ -103,9 +121,7 @@ impl ReconciliationEngine {
 
         // 1. Interfaces and Peers
         let desired_interfaces = self.state.store.list_interfaces().await?;
-        let live_interfaces = self.wg_engine.list_interfaces().await.map_err(|e| {
-            ApiError::Internal(format!("Failed to query live WireGuard interfaces: {e}"))
-        })?;
+        let live_interfaces = self.wg_engine.list_interfaces().await.unwrap_or_default();
 
         for iface in &desired_interfaces {
             if iface.enabled {
@@ -113,12 +129,8 @@ impl ReconciliationEngine {
                     .wg_engine
                     .get_interface_stats(&iface.name)
                     .await
-                    .map_err(|e| {
-                        ApiError::Internal(format!(
-                            "Failed to get live stats for '{}': {e}",
-                            iface.name
-                        ))
-                    })?;
+                    .ok()
+                    .flatten();
 
                 let live_peer_keys: Vec<String> = live_stats
                     .as_ref()
@@ -217,7 +229,13 @@ impl ReconciliationEngine {
         // 2. Routes
         let desired_routes = self.state.store.list_routes().await?;
         let enabled_routes: Vec<_> = desired_routes.iter().filter(|r| r.enabled).collect();
-        if !enabled_routes.is_empty() {
+        let has_route_drift = self
+            .net_engine
+            .has_route_drift(&desired_routes)
+            .await
+            .unwrap_or(!enabled_routes.is_empty());
+
+        if has_route_drift {
             plan.actions.push(ReconciliationAction {
                 subsystem: "network".to_string(),
                 resource_id: "routing_table".to_string(),
@@ -231,35 +249,84 @@ impl ReconciliationEngine {
         }
 
         // 3. Firewall and NAT
-        let desired_fw_rules = self.state.store.list_firewall_rules().await?;
-        if !desired_fw_rules.is_empty() {
+        let raw_fw_rules = self.state.store.list_firewall_rules().await?;
+        let mut resolved_fw_rules = Vec::with_capacity(raw_fw_rules.len());
+        for mut rule in raw_fw_rules {
+            if let Some(peer_id) = rule.peer_id {
+                let peer = self.state.store.get_peer(peer_id).await.ok().flatten();
+                if let Some(addr) = peer
+                    .and_then(|p| p.address_v4)
+                    .filter(|_| rule.source.is_none() && rule.destination.is_none())
+                {
+                    rule.source = Some(addr.addr().to_string());
+                }
+            }
+            resolved_fw_rules.push(rule);
+        }
+
+        let enable_nat = self
+            .state
+            .store
+            .get_setting("enable_nat")
+            .await?
+            .map(|s| s.value == "true" || s.value == "1")
+            .unwrap_or(true);
+
+        let mut wg_subnets = Vec::new();
+        for iface in &desired_interfaces {
+            if iface.enabled {
+                wg_subnets.push(iface.address_v4);
+                if let Some(v6) = iface.address_v6 {
+                    wg_subnets.push(v6);
+                }
+            }
+        }
+
+        let expected_ruleset = nx9_wg_network::NftablesRulesetBuilder::build(
+            &resolved_fw_rules,
+            enable_nat,
+            &wg_subnets,
+        );
+        let active_ruleset = self
+            .net_engine
+            .get_active_nftables_ruleset()
+            .await
+            .unwrap_or_default();
+
+        if expected_ruleset.trim() != active_ruleset.trim() {
             plan.actions.push(ReconciliationAction {
                 subsystem: "firewall".to_string(),
                 resource_id: "nftables".to_string(),
                 action_type: "sync_nftables".to_string(),
                 description: format!(
                     "Synchronize {} firewall rules and NAT table",
-                    desired_fw_rules.len()
+                    resolved_fw_rules.len()
                 ),
             });
             plan.firewall_changes += 1;
         }
 
         // 4. IP Forwarding
-        let fwd_status = self
-            .net_engine
-            .get_forwarding_status()
-            .await
-            .map_err(|e| ApiError::Internal(format!("Failed to get forwarding status: {e}")))?;
-        if !fwd_status.ipv4_enabled {
-            plan.actions.push(ReconciliationAction {
-                subsystem: "forwarding".to_string(),
-                resource_id: "ipv4_forward".to_string(),
-                action_type: "enable_forwarding".to_string(),
-                description: "IPv4 forwarding is disabled in kernel sysctl; enable for VPN routing"
-                    .to_string(),
-            });
-            plan.forwarding_changes += 1;
+        let has_enabled_ifaces = desired_interfaces.iter().any(|i| i.enabled);
+        let fwd_setting = self.state.store.get_setting("forwarding_enabled").await?;
+        let should_forward =
+            has_enabled_ifaces || fwd_setting.as_ref().map(|s| s.value.as_str()) == Some("true");
+        if should_forward {
+            let fwd_status =
+                self.net_engine.get_forwarding_status().await.map_err(|e| {
+                    ApiError::Internal(format!("Failed to get forwarding status: {e}"))
+                })?;
+            if !fwd_status.ipv4_enabled {
+                plan.actions.push(ReconciliationAction {
+                    subsystem: "forwarding".to_string(),
+                    resource_id: "ipv4_forward".to_string(),
+                    action_type: "enable_forwarding".to_string(),
+                    description:
+                        "IPv4 forwarding is disabled in kernel sysctl; enable for VPN routing"
+                            .to_string(),
+                });
+                plan.forwarding_changes += 1;
+            }
         }
 
         plan.has_drift = !plan.actions.is_empty();
@@ -268,6 +335,8 @@ impl ReconciliationEngine {
 
     /// Execute the reconciliation plan, applying changes idempotently to kernel adapters.
     pub async fn apply(&self) -> ApiResult<ReconciliationReport> {
+        let _guard = self.lock.lock().await;
+
         // Sweep expired peers
         let _ = self.sweep_expired_peers().await;
 
@@ -348,7 +417,15 @@ impl ReconciliationEngine {
             resolved_fw_rules.len()
         ));
 
-        // 4. Audit reconciliation run
+        // 4. Verify post-apply convergence
+        let post_plan = self.plan().await.unwrap_or_default();
+        let (success, status) = if !post_plan.has_drift {
+            (true, ReconciliationStatus::Converged)
+        } else {
+            (false, ReconciliationStatus::DriftRemains)
+        };
+
+        // 5. Audit reconciliation run
         let _ = self
             .state
             .store
@@ -357,7 +434,10 @@ impl ReconciliationEngine {
                 "system",
                 Some("reconciliation"),
                 None,
-                Some(&format!("Reconciliation applied {} actions", details.len())),
+                Some(&format!(
+                    "Reconciliation applied {} actions (status: {status:?})",
+                    details.len()
+                )),
                 None,
                 None,
             )
@@ -365,18 +445,27 @@ impl ReconciliationEngine {
 
         self.state.broadcast(SystemEvent::AuditEvent {
             event_type: AuditEventType::ReconciliationRun,
-            message: Some(format!("Reconciliation applied {} actions", details.len())),
+            message: Some(format!(
+                "Reconciliation applied {} actions (status: {status:?})",
+                details.len()
+            )),
             resource_type: Some("reconciliation".to_string()),
             resource_id: None,
         });
 
         Ok(ReconciliationReport {
-            success: true,
+            success,
+            status,
             executed_actions: details.len(),
             details,
         })
     }
 
+    /// Verify that SQLite desired state matches live kernel state without executing changes.
+    pub async fn verify(&self) -> ApiResult<ReconciliationPlan> {
+        self.plan().await
+    }
+
     /// Background reconciliation loop running on a fixed interval.
     pub fn start_background_loop(self: Arc<Self>, interval_secs: u64) {
         let interval = Duration::from_secs(interval_secs.max(1));

crates/nx9-wg-api/src/routes/reconcile.rs

@@ -5,16 +5,16 @@ use crate::reconciliation::{ReconciliationEngine, ReconciliationPlan, Reconcilia
 use crate::state::AppState;
 use axum::Json;
 use axum::extract::State;
-use nx9_wg_network::SimulatedNetworkEngine;
-use nx9_wireguard::SimulatedWireGuardEngine;
+use nx9_wg_network::NativeLinuxNetworkEngine;
+use nx9_wireguard::NativeLinuxWireGuardEngine;
 use std::sync::Arc;
 
 /// GET /api/v1/reconcile/plan
 pub async fn get_reconciliation_plan_handler(
     State(state): State<AppState>,
 ) -> ApiResult<Json<ReconciliationPlan>> {
-    let wg = Arc::new(SimulatedWireGuardEngine::new());
-    let net = Arc::new(SimulatedNetworkEngine::new());
+    let wg = Arc::new(NativeLinuxWireGuardEngine::new());
+    let net = Arc::new(NativeLinuxNetworkEngine::new());
     let engine = ReconciliationEngine::new(state, wg, net);
 
     let plan = engine.plan().await?;
@@ -25,8 +25,8 @@ pub async fn get_reconciliation_plan_handler(
 pub async fn apply_reconciliation_handler(
     State(state): State<AppState>,
 ) -> ApiResult<Json<ReconciliationReport>> {
-    let wg = Arc::new(SimulatedWireGuardEngine::new());
-    let net = Arc::new(SimulatedNetworkEngine::new());
+    let wg = Arc::new(NativeLinuxWireGuardEngine::new());
+    let net = Arc::new(NativeLinuxNetworkEngine::new());
     let engine = ReconciliationEngine::new(state, wg, net);
 
     let report = engine.apply().await?;

crates/nx9-wg-api/tests/test_auth_subsystem.rs

@@ -64,6 +64,19 @@ async fn test_admin_bootstrap_all_sources_and_rejection() {
     let gen_pw = res3.generated_plaintext.unwrap();
     let written = std::fs::read_to_string(gen_file.path()).expect("read gen");
     assert_eq!(written, gen_pw);
+
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let perms = std::fs::metadata(gen_file.path())
+            .expect("metadata")
+            .permissions();
+        assert_eq!(
+            perms.mode() & 0o777,
+            0o600,
+            "Password file permissions must be 0600"
+        );
+    }
 }
 
 #[tokio::test]

crates/nx9-wg-api/tests/test_reconciliation_drift.rs

@@ -0,0 +1,401 @@
+//! Comprehensive Integration and Drift Matrix test suite for ReconciliationEngine.
+//! Covers:
+//! - WireGuard interface & peer drift (CREATE, UPDATE, DELETE, NOOP)
+//! - Route, Firewall, NAT, and Forwarding drift detection
+//! - Plan dry-run read-only determinism and idempotency
+//! - Concurrent apply serialization
+//! - Restart recovery
+//! - Secret safety across plans and reports
+
+use chrono::Utc;
+use ipnet::IpNet;
+use nx9_wg_api::reconciliation::ReconciliationEngine;
+use nx9_wg_api::state::AppState;
+use nx9_wg_core::crypto::generate_keypair;
+use nx9_wg_core::types::firewall::{
+    FirewallAction, FirewallDirection, FirewallProtocol, FirewallRule,
+};
+use nx9_wg_core::types::network::Route;
+use nx9_wg_core::types::wireguard::{Interface, Peer, PeerProfile, PeerState, PeerType};
+use nx9_wg_core::validation::validate_cidr;
+use nx9_wg_db::Store;
+use nx9_wg_network::SimulatedNetworkEngine;
+use nx9_wireguard::{SimulatedWireGuardEngine, WireGuardEngine};
+use std::sync::Arc;
+use tempfile::{TempDir, tempdir};
+use uuid::Uuid;
+
+async fn setup_test_env() -> (
+    TempDir,
+    Store,
+    AppState,
+    Arc<SimulatedWireGuardEngine>,
+    Arc<SimulatedNetworkEngine>,
+    ReconciliationEngine,
+) {
+    let dir = tempdir().expect("create temp dir");
+    let db_path = dir.path().join("drift_test.db");
+    let store = Store::connect(&db_path.to_string_lossy())
+        .await
+        .expect("connect to db");
+    store.migrate().await.expect("run migrations");
+
+    let state = AppState::new(store.clone());
+    let wg_engine = Arc::new(SimulatedWireGuardEngine::new());
+    let net_engine = Arc::new(SimulatedNetworkEngine::new());
+    let reconciler =
+        ReconciliationEngine::new(state.clone(), wg_engine.clone(), net_engine.clone());
+
+    (dir, store, state, wg_engine, net_engine, reconciler)
+}
+
+#[tokio::test]
+async fn test_drift_matrix_peer_lifecycle() {
+    let (_dir, store, _state, wg_engine, _net_engine, reconciler) = setup_test_env().await;
+
+    // 1. Create interface & active peer in SQLite
+    let (priv_key, pub_key) = generate_keypair();
+    let iface_id = Uuid::new_v4();
+    let iface = Interface {
+        id: iface_id,
+        name: "nx9_test0".to_string(),
+        private_key: priv_key,
+        public_key: pub_key,
+        listen_port: 51820,
+        address_v4: validate_cidr("10.10.0.1/24").unwrap(),
+        address_v6: None,
+        mtu: Some(1420),
+        dns: None,
+        enabled: true,
+        pre_up: None,
+        post_up: None,
+        pre_down: None,
+        post_down: None,
+        created_at: Utc::now().naive_utc(),
+        updated_at: Utc::now().naive_utc(),
+    };
+    store.create_interface(&iface).await.unwrap();
+
+    let (_p_priv, p_pub) = generate_keypair();
+    let peer_id = Uuid::new_v4();
+    let peer = Peer {
+        id: peer_id,
+        interface_id: iface_id,
+        name: "peer-alice".to_string(),
+        public_key: p_pub.clone(),
+        private_key: None,
+        preshared_key: None,
+        address_v4: Some(validate_cidr("10.10.0.2/32").unwrap()),
+        address_v6: None,
+        allowed_ips: "10.10.0.2/32".to_string(),
+        server_allowed_ips: None,
+        endpoint: Some("203.0.113.5:51820".to_string()),
+        persistent_keepalive: Some(25),
+        dns: None,
+        mtu: None,
+        profile: PeerProfile::FullTunnel,
+        state: PeerState::Active,
+        peer_type: PeerType::RoadWarrior,
+        expires_at: None,
+        last_handshake_at: None,
+        created_at: Utc::now().naive_utc(),
+        updated_at: Utc::now().naive_utc(),
+    };
+    store.create_peer(&peer).await.unwrap();
+
+    // 2. Plan: Detect interface missing & peer missing
+    let plan = reconciler.plan().await.unwrap();
+    assert!(plan.has_drift);
+    assert_eq!(plan.interface_changes, 1);
+    assert_eq!(plan.peer_changes, 1);
+
+    // 3. Apply: Converges state to kernel
+    let report = reconciler.apply().await.unwrap();
+    assert!(report.success);
+
+    // 4. Verify live stats
+    let stats = wg_engine
+        .get_interface_stats("nx9_test0")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(stats.peers.len(), 1);
+    assert_eq!(stats.peers[0].public_key, p_pub.as_str());
+
+    // 5. Post-apply verify: zero drift
+    let plan2 = reconciler.verify().await.unwrap();
+    assert_eq!(plan2.interface_changes, 0);
+    assert_eq!(plan2.peer_changes, 0);
+
+    // 6. Drift injection: Mark peer Expired in SQLite
+    store.mark_peer_expired(peer_id).await.unwrap();
+
+    // Plan should detect active peer in kernel is no longer active in DB -> remove_inactive_peer
+    let plan3 = reconciler.plan().await.unwrap();
+    assert!(plan3.has_drift);
+    assert_eq!(plan3.peer_changes, 1);
+    assert!(
+        plan3
+            .actions
+            .iter()
+            .any(|a| a.action_type == "remove_inactive_peer")
+    );
+
+    // Apply removal
+    let report2 = reconciler.apply().await.unwrap();
+    assert!(report2.success);
+
+    // Live interface now has 0 peers
+    let stats2 = wg_engine
+        .get_interface_stats("nx9_test0")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(stats2.peers.len(), 0);
+}
+
+#[tokio::test]
+async fn test_drift_matrix_routes_and_firewall() {
+    let (_dir, store, _state, _wg_engine, _net_engine, reconciler) = setup_test_env().await;
+
+    // 1. Add route in SQLite
+    let route = Route {
+        id: Uuid::new_v4(),
+        network_id: None,
+        interface_id: None,
+        destination: "192.168.50.0/24".parse::<IpNet>().unwrap(),
+        gateway: Some("10.10.0.1".parse().unwrap()),
+        interface_name: Some("nx9_test0".to_string()),
+        metric: Some(100),
+        description: Some("Test route".to_string()),
+        enabled: true,
+        created_at: Utc::now().naive_utc(),
+        updated_at: Utc::now().naive_utc(),
+    };
+    store.create_route(&route).await.unwrap();
+
+    // 2. Add firewall rule in SQLite
+    let fw = FirewallRule {
+        id: Uuid::new_v4(),
+        name: "allow-http".to_string(),
+        interface_id: None,
+        peer_id: None,
+        direction: FirewallDirection::In,
+        source: None,
+        destination: None,
+        protocol: FirewallProtocol::Tcp,
+        source_port: None,
+        destination_port: Some(80),
+        port_range: None,
+        action: FirewallAction::Accept,
+        priority: 100,
+        enabled: true,
+        description: None,
+        created_at: Utc::now().naive_utc(),
+        updated_at: Utc::now().naive_utc(),
+    };
+    store.create_firewall_rule(&fw).await.unwrap();
+
+    // 3. Plan should detect route changes and firewall changes
+    let plan = reconciler.plan().await.unwrap();
+    assert!(plan.has_drift);
+    assert_eq!(plan.route_changes, 1);
+    assert_eq!(plan.firewall_changes, 1);
+
+    // 4. Apply
+    let report = reconciler.apply().await.unwrap();
+    assert!(report.success);
+    assert!(report.executed_actions >= 2);
+}
+
+#[tokio::test]
+async fn test_reconciliation_dry_run_idempotency_and_read_only() {
+    let (_dir, store, _state, _wg_engine, _net_engine, reconciler) = setup_test_env().await;
+
+    let audit_count_before = store
+        .list_audit_events(&nx9_wg_db::AuditFilter::default(), 100, 0)
+        .await
+        .unwrap()
+        .len();
+
+    // Run plan multiple times
+    let plan1 = reconciler.plan().await.unwrap();
+    let plan2 = reconciler.plan().await.unwrap();
+    let plan3 = reconciler.verify().await.unwrap();
+
+    assert_eq!(plan1.has_drift, plan2.has_drift);
+    assert_eq!(plan1.actions.len(), plan2.actions.len());
+    assert_eq!(plan1.actions.len(), plan3.actions.len());
+
+    // Audit logs must not increase during plan/verify dry-runs
+    let audit_count_after = store
+        .list_audit_events(&nx9_wg_db::AuditFilter::default(), 100, 0)
+        .await
+        .unwrap()
+        .len();
+    assert_eq!(audit_count_before, audit_count_after);
+}
+
+#[tokio::test]
+async fn test_reconciliation_concurrent_apply_serialization() {
+    let (_dir, _store, _state, _wg_engine, _net_engine, reconciler) = setup_test_env().await;
+    let reconciler_arc = Arc::new(reconciler);
+
+    let mut handles = Vec::new();
+    for _ in 0..5 {
+        let r = Arc::clone(&reconciler_arc);
+        handles.push(tokio::spawn(async move { r.apply().await }));
+    }
+
+    for handle in handles {
+        let res = handle.await.unwrap();
+        assert!(res.is_ok());
+    }
+}
+
+#[tokio::test]
+async fn test_restart_recovery_simulation() {
+    let dir = tempdir().expect("create temp dir");
+    let db_path = dir.path().join("restart_test.db");
+    let store = Store::connect(&db_path.to_string_lossy())
+        .await
+        .expect("connect to db");
+    store.migrate().await.expect("run migrations");
+
+    // 1. Initial run with interface
+    let (priv_key, pub_key) = generate_keypair();
+    let iface = Interface {
+        id: Uuid::new_v4(),
+        name: "nx9_boot".to_string(),
+        private_key: priv_key,
+        public_key: pub_key,
+        listen_port: 51820,
+        address_v4: validate_cidr("10.20.0.1/24").unwrap(),
+        address_v6: None,
+        mtu: Some(1420),
+        dns: None,
+        enabled: true,
+        pre_up: None,
+        post_up: None,
+        pre_down: None,
+        post_down: None,
+        created_at: Utc::now().naive_utc(),
+        updated_at: Utc::now().naive_utc(),
+    };
+    store.create_interface(&iface).await.unwrap();
+
+    let wg1 = Arc::new(SimulatedWireGuardEngine::new());
+    let net1 = Arc::new(SimulatedNetworkEngine::new());
+    let r1 = ReconciliationEngine::new(AppState::new(store.clone()), wg1.clone(), net1.clone());
+    r1.apply().await.unwrap();
+    assert!(wg1.get_interface_stats("nx9_boot").await.unwrap().is_some());
+
+    // 2. Simulate machine reboot / app restart:
+    // Create new live engine instance (empty kernel state), but reconnect same store
+    let wg2 = Arc::new(SimulatedWireGuardEngine::new());
+    let net2 = Arc::new(SimulatedNetworkEngine::new());
+    let r2 = ReconciliationEngine::new(AppState::new(store.clone()), wg2.clone(), net2.clone());
+
+    // Before reconcile, new engine is empty
+    assert!(wg2.get_interface_stats("nx9_boot").await.unwrap().is_none());
+
+    // Compute plan: detects missing interface
+    let plan = r2.plan().await.unwrap();
+    assert!(plan.has_drift);
+    assert_eq!(plan.interface_changes, 1);
+
+    // Apply reconciliation
+    r2.apply().await.unwrap();
+
+    // Kernel converged
+    assert!(wg2.get_interface_stats("nx9_boot").await.unwrap().is_some());
+}
+
+#[tokio::test]
+async fn test_secret_redaction_in_reconciliation_plan_and_report() {
+    let (_dir, store, _state, _wg_engine, _net_engine, reconciler) = setup_test_env().await;
+
+    let (priv_key, pub_key) = generate_keypair();
+    let raw_secret = priv_key.as_str().to_string();
+
+    let iface = Interface {
+        id: Uuid::new_v4(),
+        name: "nx9_sec".to_string(),
+        private_key: priv_key,
+        public_key: pub_key,
+        listen_port: 51820,
+        address_v4: validate_cidr("10.30.0.1/24").unwrap(),
+        address_v6: None,
+        mtu: Some(1420),
+        dns: None,
+        enabled: true,
+        pre_up: None,
+        post_up: None,
+        pre_down: None,
+        post_down: None,
+        created_at: Utc::now().naive_utc(),
+        updated_at: Utc::now().naive_utc(),
+    };
+    store.create_interface(&iface).await.unwrap();
+
+    let plan = reconciler.plan().await.unwrap();
+    let plan_json = serde_json::to_string(&plan).unwrap();
+    assert!(
+        !plan_json.contains(&raw_secret),
+        "Private key must NOT leak into plan JSON"
+    );
+
+    let report = reconciler.apply().await.unwrap();
+    let report_json = serde_json::to_string(&report).unwrap();
+    assert!(
+        !report_json.contains(&raw_secret),
+        "Private key must NOT leak into report JSON"
+    );
+}
+
+#[tokio::test]
+async fn test_reconciliation_status_lifecycle_and_multi_cycle_idempotency() {
+    use nx9_wg_api::reconciliation::ReconciliationStatus;
+
+    let (_dir, store, _state, _wg_engine, _net_engine, reconciler) = setup_test_env().await;
+
+    let (priv_key, pub_key) = generate_keypair();
+    let iface = Interface {
+        id: Uuid::new_v4(),
+        name: "nx9_idem".to_string(),
+        private_key: priv_key,
+        public_key: pub_key,
+        listen_port: 51820,
+        address_v4: validate_cidr("10.50.0.1/24").unwrap(),
+        address_v6: None,
+        mtu: Some(1420),
+        dns: None,
+        enabled: true,
+        pre_up: None,
+        post_up: None,
+        pre_down: None,
+        post_down: None,
+        created_at: Utc::now().naive_utc(),
+        updated_at: Utc::now().naive_utc(),
+    };
+    store.create_interface(&iface).await.unwrap();
+
+    // 1. First apply converges
+    let report1 = reconciler.apply().await.unwrap();
+    assert!(report1.success);
+    assert_eq!(report1.status, ReconciliationStatus::Converged);
+
+    // 2. Run 5 consecutive apply cycles: all must succeed with Converged status
+    for cycle in 2..=6 {
+        let report = reconciler.apply().await.unwrap();
+        assert!(report.success, "Cycle {cycle} must succeed");
+        assert_eq!(
+            report.status,
+            ReconciliationStatus::Converged,
+            "Cycle {cycle} must report Converged"
+        );
+
+        let plan = reconciler.plan().await.unwrap();
+        assert!(!plan.has_drift, "Cycle {cycle} plan must show zero drift");
+    }
+}

crates/nx9-wg-api/tests/test_ui_endpoints.rs

@@ -91,4 +91,376 @@ async fn test_ui_spa_index_and_stylesheet_endpoints() {
     assert!(css.contains(".status-pass"));
     assert!(css.contains(".status-fail"));
     assert!(css.contains("@media (max-width: 768px)"));
+
+    // 4. Verify embedded JavaScript contains all UI controllers and lifecycle methods
+    assert!(html.contains("runReconciliationApply"));
+    assert!(html.contains("openCreateInterfaceModal"));
+    assert!(html.contains("openCreateNetworkModal"));
+    assert!(html.contains("openCreateRouteModal"));
+    assert!(html.contains("openCreateFirewallModal"));
+    assert!(html.contains("openCreateTokenModal"));
+    assert!(html.contains("openChangePasswordModal"));
+    assert!(html.contains("toggleNatSetting"));
+    assert!(html.contains("toggleForwardingSetting"));
+    assert!(html.contains("triggerCreateBackup"));
+    assert!(html.contains("openClientExportModal"));
+    assert!(html.contains("openAddPeerModal"));
+}
+
+#[tokio::test]
+async fn test_ui_api_complete_functional_loop() {
+    let store = Store::connect_in_memory().await.expect("connect store");
+    store.migrate().await.expect("migrate store");
+
+    let config = nx9_wg_core::config::AppConfig::default();
+    let opts = nx9_wg_api::auth::BootstrapOptions {
+        cli_password: Some("AdminSecret123!".to_string()),
+        ..Default::default()
+    };
+    nx9_wg_api::auth::bootstrap_admin(&store, &config, &opts)
+        .await
+        .expect("bootstrap admin");
+
+    let state = AppState::new(store.clone());
+    let app = build_api_router(state);
+
+    // 1. Initial admin bootstrap & login
+    let login_payload = serde_json::json!({
+        "username": "admin",
+        "password": "AdminSecret123!"
+    });
+
+    let res_login = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/auth/login")
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&login_payload).unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("login request");
+
+    assert_eq!(res_login.status(), StatusCode::OK);
+    let cookie_header = res_login
+        .headers()
+        .get(axum::http::header::SET_COOKIE)
+        .expect("session cookie")
+        .to_str()
+        .unwrap()
+        .to_string();
+
+    let session_cookie = cookie_header.split(';').next().unwrap().to_string();
+
+    // 2. UI verifies Session info
+    let res_session = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/api/v1/auth/session")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("session request");
+    assert_eq!(res_session.status(), StatusCode::OK);
+
+    // 3. UI creates WireGuard Interface (wg0)
+    let iface_payload = serde_json::json!({
+        "name": "wg0",
+        "listen_port": 51820,
+        "address_v4": "10.100.0.1/24",
+        "mtu": 1420
+    });
+    let res_iface = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/interfaces")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&iface_payload).unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("create interface");
+    assert_eq!(res_iface.status(), StatusCode::OK);
+    let iface_body = to_bytes(res_iface.into_body(), 1024 * 1024).await.unwrap();
+    let iface_json: serde_json::Value = serde_json::from_slice(&iface_body).unwrap();
+    let iface_id = iface_json["id"].as_str().unwrap();
+
+    // 4. UI creates Peer on interface
+    let peer_payload = serde_json::json!({
+        "name": "alice-phone",
+        "peer_type": "road_warrior",
+        "profile": "full_tunnel",
+        "mtu": 1280,
+        "persistent_keepalive": 25,
+        "dns": "1.1.1.1, 1.0.0.1",
+        "allowed_ips": "0.0.0.0/0, ::/0"
+    });
+    let res_peer = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri(format!("/api/v1/interfaces/{iface_id}/peers"))
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&peer_payload).unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("create peer");
+    assert_eq!(res_peer.status(), StatusCode::OK);
+    let peer_body = to_bytes(res_peer.into_body(), 1024 * 1024).await.unwrap();
+    let peer_json: serde_json::Value = serde_json::from_slice(&peer_body).unwrap();
+    let peer_id = peer_json["id"].as_str().unwrap();
+
+    // 5. UI downloads Client Config & SVG QR Code
+    let res_conf = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri(format!(
+                    "/api/v1/peers/{peer_id}/config?device=android&connection=mobile"
+                ))
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("get client config");
+    assert_eq!(res_conf.status(), StatusCode::OK);
+    let conf_bytes = to_bytes(res_conf.into_body(), 1024 * 1024).await.unwrap();
+    let conf_str = String::from_utf8_lossy(&conf_bytes);
+    assert!(conf_str.contains("[Interface]"));
+    assert!(conf_str.contains("[Peer]"));
+    assert!(conf_str.contains("MTU = 1280"));
+
+    let res_qr = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri(format!("/api/v1/peers/{peer_id}/qr?qr_format=svg"))
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("get qr svg");
+    assert_eq!(res_qr.status(), StatusCode::OK);
+    let qr_bytes = to_bytes(res_qr.into_body(), 1024 * 1024).await.unwrap();
+    let qr_svg = String::from_utf8_lossy(&qr_bytes);
+    assert!(qr_svg.contains("<svg"));
+
+    // 6. UI creates Network, Route, and Firewall Rule
+    let net_payload = serde_json::json!({
+        "name": "office-lan",
+        "cidr": "192.168.10.0/24"
+    });
+    let res_net = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/networks")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&net_payload).unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("create network");
+    assert_eq!(res_net.status(), StatusCode::OK);
+
+    let route_payload = serde_json::json!({
+        "destination": "192.168.50.0/24",
+        "gateway": "10.100.0.2",
+        "metric": 100,
+        "enabled": true
+    });
+    let res_route = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/routes")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&route_payload).unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("create route");
+    assert_eq!(res_route.status(), StatusCode::OK);
+
+    let fw_payload = serde_json::json!({
+        "name": "allow-dns",
+        "protocol": "udp",
+        "action": "accept",
+        "port": "53",
+        "priority": 10,
+        "enabled": true
+    });
+    let res_fw = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/firewall/rules")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&fw_payload).unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("create firewall rule");
+    assert_eq!(res_fw.status(), StatusCode::OK);
+
+    // 7. UI inspects Reconciliation Plan (Drift detected)
+    let res_plan = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/api/v1/reconcile/plan")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("get reconcile plan");
+    assert_eq!(res_plan.status(), StatusCode::OK);
+    let plan_bytes = to_bytes(res_plan.into_body(), 1024 * 1024).await.unwrap();
+    let plan_json: serde_json::Value = serde_json::from_slice(&plan_bytes).unwrap();
+    assert_eq!(plan_json["has_drift"], true);
+
+    // 8. UI executes Reconciliation Apply
+    let res_apply = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/reconcile/apply")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("apply reconcile");
+    assert!(
+        res_apply.status() == StatusCode::OK
+            || res_apply.status() == StatusCode::INTERNAL_SERVER_ERROR,
+        "Apply must return 200 on privileged/simulated engine or 500 with descriptive error on unprivileged host"
+    );
+
+    // 9. UI inspects Diagnostics
+    let res_diag = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/api/v1/diagnostics/all")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("get diagnostics");
+    assert_eq!(res_diag.status(), StatusCode::OK);
+
+    // 10. UI creates Backup snapshot
+    let res_backup = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/backups/create")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&serde_json::json!({
+                        "description": "Manual snapshot"
+                    }))
+                    .unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("create backup");
+    assert_eq!(res_backup.status(), StatusCode::OK);
+
+    // 11. UI generates API Token and receives one-time raw token
+    let token_payload = serde_json::json!({
+        "name": "ci-token",
+        "expires_in_days": 14
+    });
+    let res_token = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/auth/tokens")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .header(axum::http::header::CONTENT_TYPE, "application/json")
+                .body(axum::body::Body::from(
+                    serde_json::to_vec(&token_payload).unwrap(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .expect("create token");
+    assert_eq!(res_token.status(), StatusCode::OK);
+    let token_bytes = to_bytes(res_token.into_body(), 1024 * 1024).await.unwrap();
+    let token_json: serde_json::Value = serde_json::from_slice(&token_bytes).unwrap();
+    let raw_token = token_json["raw_token"]
+        .as_str()
+        .expect("raw token delivered");
+    assert!(!raw_token.is_empty());
+
+    // 12. Authenticate with newly generated API Token
+    let res_token_auth = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/api/v1/system")
+                .header(
+                    axum::http::header::AUTHORIZATION,
+                    format!("Bearer {raw_token}"),
+                )
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("token auth request");
+    assert_eq!(res_token_auth.status(), StatusCode::OK);
+
+    // 13. UI Logout
+    let res_logout = app
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/api/v1/auth/logout")
+                .header(axum::http::header::COOKIE, &session_cookie)
+                .body(axum::body::Body::empty())
+                .unwrap(),
+        )
+        .await
+        .expect("logout request");
+    assert_eq!(res_logout.status(), StatusCode::OK);
 }