security: fix panic on untrusted input (Phase 2 Critical)

- Replace all sqlite::open().unwrap() per-request with shared Arc<Mutex<Connection>>
- Add Mutex poisoning recovery in all cfg.lock() and db.lock() calls
- Fix std::str::from_utf8().unwrap() with safe match → 400 Bad Request
- Fix timestamp_nanos_opt().unwrap() with safe match
- Fix panic on RPC client failure (bal-pusher): error log + sleep + retry
- Fix ZMQ socket connect with retry loop (bal-pusher)
- Add ZMQ_RCVTIMEO=5000 and match recv for graceful timeout (bal-pusher)
- Fix ZMQ subscribe error with match instead of unwrap (bal-pusher)
- Add unwrap_or for all tbl_stats row fields in echo_stats (prevent NULL panic)
- Add panic_regression_tests.rs: test mutex poisoning recovery and NULL unwrap_or

All tests pass: cargo test --test panic_regression_tests + sql_injection_tests
Build verified: cargo check --bin=bal-server --bin=bal-pusher

tests/panic_regression_tests.rs

@@ -0,0 +1,43 @@
+use std::sync::{Arc, Mutex};
+use std::thread;
+use std::collections::HashMap;
+use sqlite::Connection;
+
+#[test]
+fn test_mutex_poisoning_recovery() {
+    let data = Arc::new(Mutex::new(0));
+    let c = data.clone();
+    let handle = thread::spawn(move || {
+        let _guard = c.lock(); // Acquire lock
+        panic!("test panic"); // Panic while holding the lock
+        // _guard is dropped during panic unwinding, poisoning the mutex
+    });
+    let result = handle.join();
+    assert!(result.is_err()); // Thread panicked
+
+    // Recovery: the same pattern used in bal-server.rs
+    let guard = match data.lock() {
+        Ok(g) => g,
+        Err(p) => {
+            p.into_inner() // Should not panic
+        }
+    };
+    assert_eq!(*guard, 0);
+}
+
+#[test]
+fn test_db_null_unwrap_or() {
+    let db = Connection::open(":memory:").unwrap();
+    let _ = db.execute("CREATE TABLE test_stats (report_date TEXT, chain TEXT, totals TEXT, waiting TEXT);");
+    let _ = db.execute("INSERT INTO test_stats (report_date, chain) VALUES ('2024-01-01', 'testnet');");
+    
+    let mut found_value = None;
+    let _ = db.iterate("SELECT * FROM test_stats;", |pairs| {
+        let row: HashMap<_, _> = pairs.into_iter().map(|(k,v)| (k.to_string(), v.map(|s| s))).collect();
+        let totals = row["totals"].clone().unwrap_or("0").to_string();
+        found_value = Some(totals);
+        true
+    });
+    
+    assert_eq!(found_value.unwrap(), "0");
+}