SAFE21.io/bal-server
1
0
Fork 0
forked from bitcoinafterlife/bal-server
Code Pull Requests Activity

fix: Docker support, WAL race condition, pusher panic fixes

Browse Source 
- Add multi-stage Dockerfile with tini, non-root user, healthcheck
- Fix SQLite WAL mode race between bal-server and bal-pusher (busy_timeout + retry)
- Fix tbl_stats missing UNIQUE index for ON CONFLICT clause
- Replace unwrap() panics in bal-pusher with graceful error handling
- Add docker/entrypoint.sh with BAL_PUSHER_NETWORK support
- cargo fmt across all files
This commit is contained in:
svatantrya
2026-07-16 21:24:09 -04:00
parent 56696050ec
commit fbe61a5862
8 changed files with 355 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/bin/bal-pusher.rs
View File

@@ -247,7 +247,13 @@ async fn main_result(cfg: &MyConfig, network_params: &NetworkParams) -> Result<(
info!("db open {}", &cfg.db_file);
let sqlquery = "SELECT * FROM tbl_tx WHERE network = :network AND status = :status AND ( locktime < :bestblock_height OR locktime > :locktime_threshold AND locktime < :bestblock_time);";
let query_tx = db.prepare(sqlquery).unwrap().into_iter();
let query_tx = match db.prepare(sqlquery) {
Ok(q) => q.into_iter(),
Err(e) => {
warn!("tbl_tx not ready yet (tables may not exist): {}", e);
return Ok(());
}
};
trace!("query_tx: {}", sqlquery);
trace!(":locktime_threshold: {}", LOCKTIME_THRESHOLD);
trace!(":bestblock_time: {}", average_time);
@@ -257,19 +263,28 @@ async fn main_result(cfg: &MyConfig, network_params: &NetworkParams) -> Result<(
//let query_tx = db.prepare("SELECT * FROM tbl_tx where status = :status").unwrap().into_iter();
let mut pushed_txs: Vec<String> = Vec::new();
let mut invalid_txs: std::collections::HashMap<String, String> = HashMap::new();
for row in query_tx
.bind::<&[(_, Value)]>(
&[
(":locktime_threshold", (LOCKTIME_THRESHOLD as i64).into()),
(":bestblock_time", (average_time as i64).into()),
(":bestblock_height", (bcinfo.blocks as i64).into()),
(":network", network_params.db_field.clone().into()),
(":status", 0.into()),
][..],
)
.unwrap()
.map(|row| row.unwrap())
{
for row_result in match query_tx.bind::<&[(_, Value)]>(
&[
(":locktime_threshold", (LOCKTIME_THRESHOLD as i64).into()),
(":bestblock_time", (average_time as i64).into()),
(":bestblock_height", (bcinfo.blocks as i64).into()),
(":network", network_params.db_field.clone().into()),
(":status", 0.into()),
][..],
) {
Ok(bound) => bound,
Err(e) => {
error!("Failed to bind query parameters: {}", e);
return Ok(());
}
} {
let row = match row_result {
Ok(r) => r,
Err(e) => {
warn!("Failed to read row: {}", e);
continue;
}
};
let tx = row.read::<&str, _>("tx");
let txid = row.read::<&str, _>("txid");
let locktime = row.read::<i64, _>("locktime");

106
src/bin/bal-server-actix.rs
View File

@@ -266,13 +266,15 @@ async fn echo_info(
&netconfig.address,
&remote_addr,
) {
Some(address) => return HttpResponse::Ok().json(InfoResponse {
address,
base_fee: netconfig.fixed_fee,
chain: netconfig.network.to_string(),
info: data.cfg.info.to_string(),
version: VERSION.to_string(),
}),
Some(address) => {
return HttpResponse::Ok().json(InfoResponse {
address,
base_fee: netconfig.fixed_fee,
chain: netconfig.network.to_string(),
info: data.cfg.info.to_string(),
version: VERSION.to_string(),
});
}
None => {
let next = get_next_address_index(&db, &netconfig.name, &netconfig.address);
next
@@ -281,16 +283,15 @@ async fn echo_info(
}; // lock released
// Derive address (CPU-bound, no lock held)
let derived = match new_address_from_xpub(
&netconfig.address, next_idx.1, netconfig.network
) {
Ok(address) => address,
Err(e) => {
error!("Failed to derive address from xpub: {}", e);
return HttpResponse::BadRequest()
.body(format!("Failed to derive address: {}", e));
}
};
let derived =
match new_address_from_xpub(&netconfig.address, next_idx.1, netconfig.network) {
Ok(address) => address,
Err(e) => {
error!("Failed to derive address from xpub: {}", e);
return HttpResponse::BadRequest()
.body(format!("Failed to derive address: {}", e));
}
};
// Lock #2: save the newly derived address
{
@@ -363,14 +364,46 @@ async fn echo_stats(path: web::Path<String>, data: web::Data<AppState>) -> impl
while let Ok(State::Row) = stmt.next() {
let report_date = stmt.read("report_date").unwrap_or("0".to_string());
let chain = stmt.read("chain").unwrap_or("?".to_string());
let totals = stmt.read("totals").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let waiting = stmt.read("waiting").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let sent = stmt.read("sent").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let failed = stmt.read("failed").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let waiting_profit = stmt.read("waiting_profit").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let sent_profit = stmt.read("sent_profit").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let missed_profit = stmt.read("missed_profit").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let unique_inputs = stmt.read("unique_inputs").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let totals = stmt
.read("totals")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let waiting = stmt
.read("waiting")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let sent = stmt
.read("sent")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let failed = stmt
.read("failed")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let waiting_profit = stmt
.read("waiting_profit")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let sent_profit = stmt
.read("sent_profit")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let missed_profit = stmt
.read("missed_profit")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let unique_inputs = stmt
.read("unique_inputs")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
stats.push(StatsResponse {
report_date,
chain,
@@ -403,7 +436,8 @@ async fn echo_search(body: Bytes, data: web::Data<AppState>) -> impl Responder {
};
info!("{}", strbody);
if strbody.is_empty() || strbody.len() != 64 || !strbody.chars().all(|c| c.is_ascii_hexdigit()) {
if strbody.is_empty() || strbody.len() != 64 || !strbody.chars().all(|c| c.is_ascii_hexdigit())
{
return HttpResponse::BadRequest().body("Invalid txid");
}
@@ -485,8 +519,8 @@ struct ParsedTx {
ntxid: String,
raw_hex: String, // the original line
locktime: String,
inputs: Vec<(String, String)>, // (in_txid, in_vout)
outputs: Vec<(usize, String, u64)> // (idx, script_pubkey, amount_sat)
inputs: Vec<(String, String)>, // (in_txid, in_vout)
outputs: Vec<(usize, String, u64)>, // (idx, script_pubkey, amount_sat)
}
/// Parse all transactions from the request body **without** needing the DB lock.
@@ -560,7 +594,7 @@ fn parse_request_transactions(
if known_addresses.contains(&address) {
address.clone()
} else {
continue
continue;
}
} else {
netconfig.address.clone()
@@ -658,9 +692,7 @@ async fn echo_push(
}; // lock released here
// Parse all transactions (CPU-bound, no DB needed)
let parsed = match parse_request_transactions(
strbody, req_time, netconfig, &known_addresses,
) {
let parsed = match parse_request_transactions(strbody, req_time, netconfig, &known_addresses) {
Ok(v) => v,
Err(resp) => return resp,
};
@@ -763,9 +795,15 @@ async fn echo_push(
}
sqlouts.push_str(" SELECT ?, ?, ?, ?");
pouts.push((lineout, Value::String(parsed.txid.clone())));
pouts.push((lineout + 1, Value::Integer(i64::try_from(*idx).unwrap_or(-1))));
pouts.push((
lineout + 1,
Value::Integer(i64::try_from(*idx).unwrap_or(-1)),
));
pouts.push((lineout + 2, Value::String(script.clone())));
pouts.push((lineout + 3, Value::Integer(i64::try_from(*amount).unwrap_or(0))));
pouts.push((
lineout + 3,
Value::Integer(i64::try_from(*amount).unwrap_or(0)),
));
lineout += 4;
}
}

74
src/db.rs
View File

@@ -1,7 +1,9 @@
use log::{error, info, trace};
use log::{error, info, trace, warn};
use sqlite::{Connection, Error, State, Value};
use std::collections::HashSet;
use std::path::Path;
use std::thread;
use std::time::Duration;
/// Check which txids are already present in the database in a single batch query.
/// Returns a HashSet of txids that already exist (duplicates).
@@ -10,28 +12,28 @@ pub fn check_duplicate_txids(db: &Connection, txids: &[String]) -> Result<HashSe
if txids.is_empty() {
return Ok(HashSet::new());
}
// Build a single query with all txids using IN clause placeholders
// SQLite supports up to 1000 parameters per statement, so we chunk for safety
let mut duplicates = HashSet::new();
let chunk_size = 500; // Safe chunk size for SQLite parameters
for chunk in txids.chunks(chunk_size) {
let placeholders = chunk.iter().map(|_| "?").collect::<Vec<_>>().join(",");
let sql = format!("SELECT txid FROM tbl_tx WHERE txid IN ({})", placeholders);
let mut stmt = db.prepare(sql)?;
for (i, txid) in chunk.iter().enumerate() {
stmt.bind((i + 1, Value::String(txid.clone())))?;
}
while let Ok(State::Row) = stmt.next() {
if let Ok(txid) = stmt.read::<String, _>("txid") {
duplicates.insert(txid);
}
}
}
Ok(duplicates)
}
@@ -61,12 +63,10 @@ pub fn open_db(path: &str) -> Result<Connection, String> {
];
for prefix in &forbidden {
if path_str.starts_with(prefix) {
return Err(
format!(
"Absolute database path under {} is forbidden",
prefix
)
);
return Err(format!(
"Absolute database path under {} is forbidden",
prefix
));
}
}
}
@@ -74,24 +74,48 @@ pub fn open_db(path: &str) -> Result<Connection, String> {
// If file exists, must be a regular file (not a symlink, device, etc.)
if p.exists() {
if p.is_symlink() {
return Err(
"Database path must not be a symlink".to_string()
);
return Err("Database path must not be a symlink".to_string());
}
let metadata = std::fs::metadata(p)
.map_err(|e| format!("Cannot access database file metadata: {}", e))?;
if !metadata.is_file() {
return Err(
"Database path must point to a regular file, not a directory or device".to_string()
"Database path must point to a regular file, not a directory or device".to_string(),
);
}
}
let conn = sqlite::open(path)
.map_err(|e| format!("Failed to open SQLite database: {}", e))?;
let conn = sqlite::open(path).map_err(|e| format!("Failed to open SQLite database: {}", e))?;
// Set busy timeout BEFORE WAL mode so SQLite waits instead of failing immediately.
// This handles the race where two processes (server + pusher) open the same DB
// and both try to enable WAL mode concurrently.
conn.execute("PRAGMA busy_timeout = 5000;")
.map_err(|e| format!("Failed to set busy_timeout: {}", e))?;
// Retry WAL mode up to 5 times (handles concurrent open from bal-pusher).
let mut wal_ok = false;
for attempt in 0..5 {
match conn.execute("PRAGMA journal_mode = WAL;") {
Ok(_) => {
wal_ok = true;
break;
}
Err(e) => {
warn!(
"WAL mode attempt {}/5 failed: {}, retrying in 100ms...",
attempt + 1,
e
);
thread::sleep(Duration::from_millis(100));
}
}
}
if !wal_ok {
// WAL might already be enabled by another process; this is not fatal.
warn!("Could not set WAL mode after retries — may already be enabled by another process");
}
conn.execute("PRAGMA journal_mode = WAL;")
.map_err(|e| format!("Failed to enable WAL mode: {}", e))?;
conn.execute("PRAGMA synchronous = NORMAL;")
.map_err(|e| format!("Failed to set synchronous NORMAL: {}", e))?;
@@ -102,13 +126,15 @@ pub fn open_db(path: &str) -> Result<Connection, String> {
/// in-memory lookup during transaction validation (replaces N+1 query).
pub fn get_all_addresses_by_xpub(db: &Connection, xpub: &str) -> Result<HashSet<String>, Error> {
let mut stmt = db.prepare(
"SELECT a.address FROM tbl_address a JOIN tbl_xpub x ON a.xpub = x.id WHERE x.xpub = ?"
"SELECT a.address FROM tbl_address a JOIN tbl_xpub x ON a.xpub = x.id WHERE x.xpub = ?",
)?;
stmt.bind((1, Value::String(xpub.to_string())))?;
let mut addresses = HashSet::new();
while let Ok(State::Row) = stmt.next() {
match stmt.read::<String, _>("address") {
Ok(addr) => { addresses.insert(addr); }
Ok(addr) => {
addresses.insert(addr);
}
Err(e) => {
error!("Failed to read address column: {}", e);
}
@@ -134,7 +160,9 @@ pub fn create_database(db: &Connection) {
let _ = db.execute("CREATE TABLE IF NOT EXISTS tbl_address (address TEXT PRIMARY_KEY, path TEXT NOT NULL, date_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, xpub INTEGER,remote_address TEXT);");
let _ = db.execute("CREATE TABLE IF NOT EXISTS tbl_stats (report_date TEXT, chain TEXT, totals INTEGER, waiting INTEGER, sent INTEGER, failed INTEGER, waiting_profit INTEGER, sent_profit INTEGER, missed_profit INTEGER, unique_inputs INTEGER);");
let _ = db.execute("CREATE INDEX IF NOT EXISTS idx_stats_chain ON tbl_stats(chain);");
// UNIQUE index required for ON CONFLICT(chain) DO UPDATE in calculate_stats
let _ = db.execute("DROP INDEX IF EXISTS idx_stats_chain;");
let _ = db.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_stats_chain ON tbl_stats(chain);");
let _ = db.execute("UPDATE tbl_tx set network='bitcoin' where network='mainnet');");
}