fix: Docker support, WAL race condition, pusher panic fixes

- Add multi-stage Dockerfile with tini, non-root user, healthcheck
- Fix SQLite WAL mode race between bal-server and bal-pusher (busy_timeout + retry)
- Fix tbl_stats missing UNIQUE index for ON CONFLICT clause
- Replace unwrap() panics in bal-pusher with graceful error handling
- Add docker/entrypoint.sh with BAL_PUSHER_NETWORK support
- cargo fmt across all files
This commit is contained in:
2026-07-16 21:24:09 -04:00
parent 56696050ec
commit fbe61a5862
8 changed files with 355 additions and 99 deletions

58
.dockerignore Normal file
View File

@@ -0,0 +1,58 @@
# Git
.git
.gitignore
.gitsecret
# Build artifacts
target/
# Environment files (secrets)
*.env
.env
.env.*
!.env.example
# Private keys
*.pem
*.key
!public_key.pem
!data/public_key.pem
# Database files
*.db
*.db-shm
*.db-wal
# Documentation
docs/
*.md
!README.md
# Tests
tests/
# Scripts (local dev only)
bal-server.sh
bal-pusher.sh
download_bal_db.sh
sendtx.sh
lib.sh
contrib/
update
update_codebase.txt
# Service files
*.service
# IDE
*.swp
*.swo
.vscode/
.idea/
# Misc
Cargo.lock
generate_random_ascii.sh
test/
invalid_txs/
valid_txs/

92
Dockerfile Normal file
View File

@@ -0,0 +1,92 @@
# =============================================================================
# Multi-stage Dockerfile for bal-server + bal-pusher
# Security: non-root user, minimal runtime, tini as PID 1
# =============================================================================
# ---------------------------------------------------------------------------
# Stage 1: Builder
# ---------------------------------------------------------------------------
FROM rust:1.95-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config \
libssl-dev \
libsodium-dev \
libzmq5-dev \
cmake \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /build
# Cache dependencies: copy Cargo.toml first, create dummy src to build deps
COPY Cargo.toml Cargo.lock* ./
RUN mkdir -p src/bin && \
echo 'fn main() {}' > src/bin/bal-server-actix.rs && \
echo 'fn main() {}' > src/bin/bal-pusher.rs && \
echo '' > src/lib.rs && \
echo '' > src/db.rs && \
echo '' > src/xpub.rs && \
echo '' > src/validation.rs && \
cargo build --release --bin bal-server --bin bal-pusher 2>/dev/null || true && \
rm -rf src target/release/.fingerprint target/release/deps/*bal_server*
# Copy real source and build
COPY src/ src/
RUN cargo build --release --bin bal-server --bin bal-pusher && \
strip target/release/bal-server target/release/bal-pusher
# ---------------------------------------------------------------------------
# Stage 2: Runtime
# ---------------------------------------------------------------------------
FROM debian:bookworm-slim AS runtime
# Install runtime dependencies + tini for PID 1
RUN apt-get update && apt-get install -y --no-install-recommends \
libssl3 \
libsodium23 \
libzmq5 \
libsqlite3-0 \
ca-certificates \
curl \
tini \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean
# Copy binaries from builder
COPY --from=builder /build/target/release/bal-server /usr/local/bin/bal-server
COPY --from=builder /build/target/release/bal-pusher /usr/local/bin/bal-pusher
# Create dedicated non-root user
RUN groupadd -g 1000 bal && \
useradd -u 1000 -g bal -s /usr/sbin/nologin -M bal && \
mkdir -p /var/bal /var/bal/.bitcoin && \
chown -R bal:bal /var/bal && \
chmod 700 /var/bal
# Copy entrypoint
COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh
# Use tini as PID 1 for proper signal handling
ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["/usr/local/bin/entrypoint.sh"]
# Data directory (mount as volume)
VOLUME ["/var/bal"]
# bal-server port (bind to 127.0.0.1 via env, expose for reverse proxy)
EXPOSE 9137
# Default environment (override at runtime)
ENV RUST_LOG=info \
BAL_SERVER_BIND_ADDRESS=127.0.0.1 \
BAL_SERVER_BIND_PORT=9137 \
BAL_SERVER_DB_FILE=/var/bal/bal.db \
BAL_PUSHER_DB_FILE=/var/bal/bal.db \
BAL_SERVER_URL=http://127.0.0.1:9137 \
BAL_SERVER_PUB_KEY_PATH=/var/bal/public_key.pem \
SSL_KEY_PATH=/var/bal/private_key.pem
# Health check: verify bal-server is responding
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD curl -sf http://127.0.0.1:9137/ || exit 1

25
docker/entrypoint.sh Normal file
View File

@@ -0,0 +1,25 @@
#!/bin/sh
set -e
mkdir -p /var/bal/.bitcoin
chown bal:bal /var/bal /var/bal/.bitcoin 2>/dev/null || true
PUSHER_NETWORK="${BAL_PUSHER_NETWORK:-bitcoin}"
echo "[entrypoint] Starting bal-server on ${BAL_SERVER_BIND_ADDRESS:-127.0.0.1}:${BAL_SERVER_BIND_PORT:-9137}"
echo "[entrypoint] Starting bal-pusher (network: ${PUSHER_NETWORK})"
su -s /bin/sh bal -c '/usr/local/bin/bal-server' &
SERVER_PID=$!
su -s /bin/sh bal -c "/usr/local/bin/bal-pusher ${PUSHER_NETWORK}" &
PUSHER_PID=$!
cleanup() {
echo "[entrypoint] Shutting down..."
kill $SERVER_PID $PUSHER_PID 2>/dev/null
wait
}
trap cleanup TERM INT
wait

View File

@@ -247,7 +247,13 @@ async fn main_result(cfg: &MyConfig, network_params: &NetworkParams) -> Result<(
info!("db open {}", &cfg.db_file);
let sqlquery = "SELECT * FROM tbl_tx WHERE network = :network AND status = :status AND ( locktime < :bestblock_height OR locktime > :locktime_threshold AND locktime < :bestblock_time);";
let query_tx = db.prepare(sqlquery).unwrap().into_iter();
let query_tx = match db.prepare(sqlquery) {
Ok(q) => q.into_iter(),
Err(e) => {
warn!("tbl_tx not ready yet (tables may not exist): {}", e);
return Ok(());
}
};
trace!("query_tx: {}", sqlquery);
trace!(":locktime_threshold: {}", LOCKTIME_THRESHOLD);
trace!(":bestblock_time: {}", average_time);
@@ -257,19 +263,28 @@ async fn main_result(cfg: &MyConfig, network_params: &NetworkParams) -> Result<(
//let query_tx = db.prepare("SELECT * FROM tbl_tx where status = :status").unwrap().into_iter();
let mut pushed_txs: Vec<String> = Vec::new();
let mut invalid_txs: std::collections::HashMap<String, String> = HashMap::new();
for row in query_tx
.bind::<&[(_, Value)]>(
&[
(":locktime_threshold", (LOCKTIME_THRESHOLD as i64).into()),
(":bestblock_time", (average_time as i64).into()),
(":bestblock_height", (bcinfo.blocks as i64).into()),
(":network", network_params.db_field.clone().into()),
(":status", 0.into()),
][..],
)
.unwrap()
.map(|row| row.unwrap())
{
for row_result in match query_tx.bind::<&[(_, Value)]>(
&[
(":locktime_threshold", (LOCKTIME_THRESHOLD as i64).into()),
(":bestblock_time", (average_time as i64).into()),
(":bestblock_height", (bcinfo.blocks as i64).into()),
(":network", network_params.db_field.clone().into()),
(":status", 0.into()),
][..],
) {
Ok(bound) => bound,
Err(e) => {
error!("Failed to bind query parameters: {}", e);
return Ok(());
}
} {
let row = match row_result {
Ok(r) => r,
Err(e) => {
warn!("Failed to read row: {}", e);
continue;
}
};
let tx = row.read::<&str, _>("tx");
let txid = row.read::<&str, _>("txid");
let locktime = row.read::<i64, _>("locktime");

View File

@@ -266,13 +266,15 @@ async fn echo_info(
&netconfig.address,
&remote_addr,
) {
Some(address) => return HttpResponse::Ok().json(InfoResponse {
address,
base_fee: netconfig.fixed_fee,
chain: netconfig.network.to_string(),
info: data.cfg.info.to_string(),
version: VERSION.to_string(),
}),
Some(address) => {
return HttpResponse::Ok().json(InfoResponse {
address,
base_fee: netconfig.fixed_fee,
chain: netconfig.network.to_string(),
info: data.cfg.info.to_string(),
version: VERSION.to_string(),
});
}
None => {
let next = get_next_address_index(&db, &netconfig.name, &netconfig.address);
next
@@ -281,16 +283,15 @@ async fn echo_info(
}; // lock released
// Derive address (CPU-bound, no lock held)
let derived = match new_address_from_xpub(
&netconfig.address, next_idx.1, netconfig.network
) {
Ok(address) => address,
Err(e) => {
error!("Failed to derive address from xpub: {}", e);
return HttpResponse::BadRequest()
.body(format!("Failed to derive address: {}", e));
}
};
let derived =
match new_address_from_xpub(&netconfig.address, next_idx.1, netconfig.network) {
Ok(address) => address,
Err(e) => {
error!("Failed to derive address from xpub: {}", e);
return HttpResponse::BadRequest()
.body(format!("Failed to derive address: {}", e));
}
};
// Lock #2: save the newly derived address
{
@@ -363,14 +364,46 @@ async fn echo_stats(path: web::Path<String>, data: web::Data<AppState>) -> impl
while let Ok(State::Row) = stmt.next() {
let report_date = stmt.read("report_date").unwrap_or("0".to_string());
let chain = stmt.read("chain").unwrap_or("?".to_string());
let totals = stmt.read("totals").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let waiting = stmt.read("waiting").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let sent = stmt.read("sent").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let failed = stmt.read("failed").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let waiting_profit = stmt.read("waiting_profit").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let sent_profit = stmt.read("sent_profit").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let missed_profit = stmt.read("missed_profit").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let unique_inputs = stmt.read("unique_inputs").unwrap_or("0".to_string()).parse::<i64>().unwrap_or(0);
let totals = stmt
.read("totals")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let waiting = stmt
.read("waiting")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let sent = stmt
.read("sent")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let failed = stmt
.read("failed")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let waiting_profit = stmt
.read("waiting_profit")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let sent_profit = stmt
.read("sent_profit")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let missed_profit = stmt
.read("missed_profit")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
let unique_inputs = stmt
.read("unique_inputs")
.unwrap_or("0".to_string())
.parse::<i64>()
.unwrap_or(0);
stats.push(StatsResponse {
report_date,
chain,
@@ -403,7 +436,8 @@ async fn echo_search(body: Bytes, data: web::Data<AppState>) -> impl Responder {
};
info!("{}", strbody);
if strbody.is_empty() || strbody.len() != 64 || !strbody.chars().all(|c| c.is_ascii_hexdigit()) {
if strbody.is_empty() || strbody.len() != 64 || !strbody.chars().all(|c| c.is_ascii_hexdigit())
{
return HttpResponse::BadRequest().body("Invalid txid");
}
@@ -485,8 +519,8 @@ struct ParsedTx {
ntxid: String,
raw_hex: String, // the original line
locktime: String,
inputs: Vec<(String, String)>, // (in_txid, in_vout)
outputs: Vec<(usize, String, u64)> // (idx, script_pubkey, amount_sat)
inputs: Vec<(String, String)>, // (in_txid, in_vout)
outputs: Vec<(usize, String, u64)>, // (idx, script_pubkey, amount_sat)
}
/// Parse all transactions from the request body **without** needing the DB lock.
@@ -560,7 +594,7 @@ fn parse_request_transactions(
if known_addresses.contains(&address) {
address.clone()
} else {
continue
continue;
}
} else {
netconfig.address.clone()
@@ -658,9 +692,7 @@ async fn echo_push(
}; // lock released here
// Parse all transactions (CPU-bound, no DB needed)
let parsed = match parse_request_transactions(
strbody, req_time, netconfig, &known_addresses,
) {
let parsed = match parse_request_transactions(strbody, req_time, netconfig, &known_addresses) {
Ok(v) => v,
Err(resp) => return resp,
};
@@ -763,9 +795,15 @@ async fn echo_push(
}
sqlouts.push_str(" SELECT ?, ?, ?, ?");
pouts.push((lineout, Value::String(parsed.txid.clone())));
pouts.push((lineout + 1, Value::Integer(i64::try_from(*idx).unwrap_or(-1))));
pouts.push((
lineout + 1,
Value::Integer(i64::try_from(*idx).unwrap_or(-1)),
));
pouts.push((lineout + 2, Value::String(script.clone())));
pouts.push((lineout + 3, Value::Integer(i64::try_from(*amount).unwrap_or(0))));
pouts.push((
lineout + 3,
Value::Integer(i64::try_from(*amount).unwrap_or(0)),
));
lineout += 4;
}
}

View File

@@ -1,7 +1,9 @@
use log::{error, info, trace};
use log::{error, info, trace, warn};
use sqlite::{Connection, Error, State, Value};
use std::collections::HashSet;
use std::path::Path;
use std::thread;
use std::time::Duration;
/// Check which txids are already present in the database in a single batch query.
/// Returns a HashSet of txids that already exist (duplicates).
@@ -10,28 +12,28 @@ pub fn check_duplicate_txids(db: &Connection, txids: &[String]) -> Result<HashSe
if txids.is_empty() {
return Ok(HashSet::new());
}
// Build a single query with all txids using IN clause placeholders
// SQLite supports up to 1000 parameters per statement, so we chunk for safety
let mut duplicates = HashSet::new();
let chunk_size = 500; // Safe chunk size for SQLite parameters
for chunk in txids.chunks(chunk_size) {
let placeholders = chunk.iter().map(|_| "?").collect::<Vec<_>>().join(",");
let sql = format!("SELECT txid FROM tbl_tx WHERE txid IN ({})", placeholders);
let mut stmt = db.prepare(sql)?;
for (i, txid) in chunk.iter().enumerate() {
stmt.bind((i + 1, Value::String(txid.clone())))?;
}
while let Ok(State::Row) = stmt.next() {
if let Ok(txid) = stmt.read::<String, _>("txid") {
duplicates.insert(txid);
}
}
}
Ok(duplicates)
}
@@ -61,12 +63,10 @@ pub fn open_db(path: &str) -> Result<Connection, String> {
];
for prefix in &forbidden {
if path_str.starts_with(prefix) {
return Err(
format!(
"Absolute database path under {} is forbidden",
prefix
)
);
return Err(format!(
"Absolute database path under {} is forbidden",
prefix
));
}
}
}
@@ -74,24 +74,48 @@ pub fn open_db(path: &str) -> Result<Connection, String> {
// If file exists, must be a regular file (not a symlink, device, etc.)
if p.exists() {
if p.is_symlink() {
return Err(
"Database path must not be a symlink".to_string()
);
return Err("Database path must not be a symlink".to_string());
}
let metadata = std::fs::metadata(p)
.map_err(|e| format!("Cannot access database file metadata: {}", e))?;
if !metadata.is_file() {
return Err(
"Database path must point to a regular file, not a directory or device".to_string()
"Database path must point to a regular file, not a directory or device".to_string(),
);
}
}
let conn = sqlite::open(path)
.map_err(|e| format!("Failed to open SQLite database: {}", e))?;
let conn = sqlite::open(path).map_err(|e| format!("Failed to open SQLite database: {}", e))?;
// Set busy timeout BEFORE WAL mode so SQLite waits instead of failing immediately.
// This handles the race where two processes (server + pusher) open the same DB
// and both try to enable WAL mode concurrently.
conn.execute("PRAGMA busy_timeout = 5000;")
.map_err(|e| format!("Failed to set busy_timeout: {}", e))?;
// Retry WAL mode up to 5 times (handles concurrent open from bal-pusher).
let mut wal_ok = false;
for attempt in 0..5 {
match conn.execute("PRAGMA journal_mode = WAL;") {
Ok(_) => {
wal_ok = true;
break;
}
Err(e) => {
warn!(
"WAL mode attempt {}/5 failed: {}, retrying in 100ms...",
attempt + 1,
e
);
thread::sleep(Duration::from_millis(100));
}
}
}
if !wal_ok {
// WAL might already be enabled by another process; this is not fatal.
warn!("Could not set WAL mode after retries — may already be enabled by another process");
}
conn.execute("PRAGMA journal_mode = WAL;")
.map_err(|e| format!("Failed to enable WAL mode: {}", e))?;
conn.execute("PRAGMA synchronous = NORMAL;")
.map_err(|e| format!("Failed to set synchronous NORMAL: {}", e))?;
@@ -102,13 +126,15 @@ pub fn open_db(path: &str) -> Result<Connection, String> {
/// in-memory lookup during transaction validation (replaces N+1 query).
pub fn get_all_addresses_by_xpub(db: &Connection, xpub: &str) -> Result<HashSet<String>, Error> {
let mut stmt = db.prepare(
"SELECT a.address FROM tbl_address a JOIN tbl_xpub x ON a.xpub = x.id WHERE x.xpub = ?"
"SELECT a.address FROM tbl_address a JOIN tbl_xpub x ON a.xpub = x.id WHERE x.xpub = ?",
)?;
stmt.bind((1, Value::String(xpub.to_string())))?;
let mut addresses = HashSet::new();
while let Ok(State::Row) = stmt.next() {
match stmt.read::<String, _>("address") {
Ok(addr) => { addresses.insert(addr); }
Ok(addr) => {
addresses.insert(addr);
}
Err(e) => {
error!("Failed to read address column: {}", e);
}
@@ -134,7 +160,9 @@ pub fn create_database(db: &Connection) {
let _ = db.execute("CREATE TABLE IF NOT EXISTS tbl_address (address TEXT PRIMARY_KEY, path TEXT NOT NULL, date_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, xpub INTEGER,remote_address TEXT);");
let _ = db.execute("CREATE TABLE IF NOT EXISTS tbl_stats (report_date TEXT, chain TEXT, totals INTEGER, waiting INTEGER, sent INTEGER, failed INTEGER, waiting_profit INTEGER, sent_profit INTEGER, missed_profit INTEGER, unique_inputs INTEGER);");
let _ = db.execute("CREATE INDEX IF NOT EXISTS idx_stats_chain ON tbl_stats(chain);");
// UNIQUE index required for ON CONFLICT(chain) DO UPDATE in calculate_stats
let _ = db.execute("DROP INDEX IF EXISTS idx_stats_chain;");
let _ = db.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_stats_chain ON tbl_stats(chain);");
let _ = db.execute("UPDATE tbl_tx set network='bitcoin' where network='mainnet');");
}

View File

@@ -6,32 +6,31 @@ use std::path::Path;
#[test]
fn test_open_db_blocks_traversal() {
let res = open_db("../etc/passwd");
assert!(
res.is_err(),
"Path with '..' should be rejected"
);
assert!(res.is_err(), "Path with '..' should be rejected");
let err = match res {
Err(e) => e,
Ok(_) => panic!("Expected error for traversal path"),
};
assert!(err.contains("'..'"), "Error should mention directory traversal: {}", err);
assert!(
err.contains("'..'"),
"Error should mention directory traversal: {}",
err
);
}
#[test]
fn test_open_db_blocks_forbidden_absolute() {
for path in ["/etc/passwd", "/proc/self/mem", "/dev/null", "/usr/bin/ls"] {
let res = open_db(path);
assert!(
res.is_err(),
"Absolute path {} should be rejected", path
);
assert!(res.is_err(), "Absolute path {} should be rejected", path);
let err = match res {
Err(e) => e,
Ok(_) => panic!("Expected error for forbidden path {}", path),
};
assert!(
err.contains("forbidden"),
"Error should mention forbidden prefix: {}", err
"Error should mention forbidden prefix: {}",
err
);
}
}
@@ -41,10 +40,7 @@ fn test_open_db_allows_relative() {
let test_path = "tmp_test_bal.db";
let _ = fs::remove_file(test_path);
let res = open_db(test_path);
assert!(
res.is_ok(),
"Valid relative path should be allowed"
);
assert!(res.is_ok(), "Valid relative path should be allowed");
let db = res.unwrap();
drop(db);
let _ = fs::remove_file(test_path);
@@ -61,10 +57,7 @@ fn test_open_db_wal_pragmas_set() {
let mut stmt = db.prepare("PRAGMA journal_mode;").unwrap();
if let Ok(State::Row) = stmt.next() {
let mode: String = stmt.read(0).unwrap();
assert_eq!(
mode, "wal",
"SQLite journal mode should be WAL"
);
assert_eq!(mode, "wal", "SQLite journal mode should be WAL");
} else {
panic!("Could not read journal_mode pragma");
}
@@ -84,17 +77,15 @@ fn test_open_db_rejects_symlink() {
fs::soft_link(real, link).unwrap();
let res = open_db(link);
assert!(
res.is_err(),
"Symlink DB path should be rejected"
);
assert!(res.is_err(), "Symlink DB path should be rejected");
let err = match res {
Err(e) => e,
Ok(_) => panic!("Expected error for symlink"),
};
assert!(
err.contains("symlink"),
"Error should mention symlink: {}", err
"Error should mention symlink: {}",
err
);
let _ = fs::remove_file(real);

View File

@@ -10,15 +10,21 @@ fn setup_db_with_xpub() -> sqlite::Connection {
"CREATE TABLE tbl_address (address TEXT PRIMARY KEY, path TEXT, xpub INTEGER, remote_address TEXT);"
);
// Insert test xpub
let mut stmt = db.prepare("INSERT INTO tbl_xpub(id, network, xpub) VALUES(?, ?, ?);").unwrap();
let mut stmt = db
.prepare("INSERT INTO tbl_xpub(id, network, xpub) VALUES(?, ?, ?);")
.unwrap();
stmt.bind((1, Value::Integer(1))).unwrap();
stmt.bind((2, Value::String("testnet".to_string()))).unwrap();
stmt.bind((3, Value::String("tpub_test".to_string()))).unwrap();
stmt.bind((2, Value::String("testnet".to_string())))
.unwrap();
stmt.bind((3, Value::String("tpub_test".to_string())))
.unwrap();
let _ = stmt.next();
drop(stmt);
// Insert test addresses
for addr in ["addr1", "addr2", "addr3"] {
let mut stmt = db.prepare("INSERT INTO tbl_address(address, path, xpub) VALUES(?, ?, ?);").unwrap();
let mut stmt = db
.prepare("INSERT INTO tbl_address(address, path, xpub) VALUES(?, ?, ?);")
.unwrap();
stmt.bind((1, Value::String(addr.to_string()))).unwrap();
stmt.bind((2, Value::String("m/0/1".to_string()))).unwrap();
stmt.bind((3, Value::Integer(1))).unwrap();
@@ -53,7 +59,10 @@ fn test_network_unknown_returns_404() {
for n in networks {
assert!(networks.contains(&n), "{} should be a valid network", n);
}
assert!(!networks.contains(&"attacker"), "attacker should not be a valid network");
assert!(
!networks.contains(&"attacker"),
"attacker should not be a valid network"
);
}
#[test]