feat(pusher): add opt-in v2 WeTime ping signing
- Add WELIST_PING_SIG_VERSION (1 legacy, 2 canonical JSON); default 1
- Sign WELIST-PING-V2\n + compact sorted JSON covering url, chain, height,
median_time, last_block_hash, info, base_fee, version and sig_version
- Fetch info/base_fee/version from the local /{chain}/info endpoint and sign them
- Fall back to null metadata on fetch failure; skip the fetch for onion URLs
- Normalize the advertised URL and bound metadata to match the WeTime service
- Activate v2 in bal-pusher.sh and document the TXT record setup
- Add unit tests for canonical payloads, URL normalization, metadata and onion detection
Includes previously uncommitted local deployment hardening:
- pg-server.sh configurable loopback bind address, volume and recreate handling
- .env loading in bal-server.sh and bal-pusher.sh
- second bal-server/bal-pusher development scripts
This commit is contained in:
10
.env.example
10
.env.example
@@ -37,6 +37,12 @@ BAL_SERVER_ACTIX_MAX_CONNECTIONS=100
|
||||
BAL_PG_PASSWORD=change_me
|
||||
BAL_PG_USER=bal
|
||||
BAL_PG_DB=baldb
|
||||
# Docker volume that stores the database, so the container can be recreated
|
||||
# without losing data.
|
||||
BAL_PG_VOLUME=bal-pg-data
|
||||
# Interface on which the container publishes port 5432. Keep 127.0.0.1 so the
|
||||
# database listens on loopback only; 0.0.0.0 would expose it on every interface.
|
||||
BAL_PG_BIND_ADDR=127.0.0.1
|
||||
|
||||
# === bal-pusher ===
|
||||
BAL_SERVER_DB_FILE=/var/bal/bal.db
|
||||
@@ -45,6 +51,10 @@ BAL_PUSHER_SEND_STATS=false
|
||||
BAL_SERVER_URL=http://127.0.0.1:9137
|
||||
SSL_KEY_PATH=/var/bal/private_key.pem
|
||||
WELIST_SERVER_URL=https://welist.bitcoin-after.life
|
||||
# Sign WeTime pings with the v2 canonical JSON scheme. Requires a DNSSEC-
|
||||
# validated `_welist-key.<host>` TXT record (or a v3 onion address). Omit or
|
||||
# set to 1 for the legacy scheme.
|
||||
WELIST_PING_SIG_VERSION=2
|
||||
|
||||
# ZMQ endpoints per network (default: localhost only)
|
||||
BAL_PUSHER_BITCOIN_ZMQ_HASHBLOCK=tcp://127.0.0.1:28332
|
||||
|
||||
@@ -148,6 +148,7 @@ If no network is specified, defaults to `bitcoin`.
|
||||
| `BAL_SERVER_URL` | URL of bal-server (for stats reporting). | - |
|
||||
| `SSL_KEY_PATH` | Ed25519 private key for signing stats reports. | `private_key.pem` |
|
||||
| `WELIST_SERVER_URL` | Welist server URL. | `https://welist.bitcoin-after.life` |
|
||||
| `WELIST_PING_SIG_VERSION` | WeTime ping signature scheme (`1` legacy, `2` canonical JSON). | `1` |
|
||||
|
||||
### Per-network configuration
|
||||
|
||||
|
||||
44
bal-pusher-2.sh
Executable file
44
bal-pusher-2.sh
Executable file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Second bal-pusher instance: talks to the second bal-server and database.
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
. "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
# Load local, gitignored environment overrides (e.g. BAL_PG_PASSWORD).
|
||||
if [ -f "$SCRIPT_DIR/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "$SCRIPT_DIR/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
WORKING_DIR=$(pwd)
|
||||
|
||||
export RUST_LOG=trace
|
||||
|
||||
export BAL_SERVER_DB_FILE="$WORKING_DIR/bal2.db"
|
||||
export BAL_SERVER_DB_BACKEND="postgresql"
|
||||
#export BAL_PUSHER_BITCOIN_COOKIE_FILE=/~/.bitcoin/.cookie
|
||||
#export BAL_PUSHER_REGTEST_COOKIE_FILE=/~/.bitcoin/regtest/.cookie
|
||||
#export BAL_PUSHER_TESTNET_COOKIE_FILE=/~/.bitcoin/testnet3/.cookie
|
||||
#export BAL_PUSHER_SIGNET_COOKIE_FILE=/~/.bitcoin/signet/.cookie
|
||||
|
||||
export BAL_PUSHER_REGTEST_ZMQ_HASHBLOCK=tcp://127.0.0.1:21332
|
||||
# Only the first pusher reports executor stats, to avoid double reporting for
|
||||
# the same executor key on the welist service.
|
||||
export BAL_PUSHER_SEND_STATS=false
|
||||
export WELIST_SERVER_URL=http://localhost:8086
|
||||
export WELIST_SKIP_URL_VALIDATION=true
|
||||
export BAL_SERVER_URL="http://127.0.0.1:9134"
|
||||
export SSL_KEY_PATH="$WORKING_DIR/private_key.pem"
|
||||
|
||||
# Instance-specific settings must win over .env, which defines BAL_PG_DB=baldb.
|
||||
export BAL_PG_DB="baldb2"
|
||||
unset BAL_SERVER_PG_DSN
|
||||
|
||||
if [ "$BAL_SERVER_DB_BACKEND" = "postgresql" ]; then
|
||||
. "$SCRIPT_DIR/pg-server.sh"
|
||||
fi
|
||||
|
||||
cargo run --bin=bal-pusher regtest --features=pusher
|
||||
@@ -4,6 +4,14 @@ set -e
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
. "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
# Load local, gitignored environment overrides (e.g. BAL_PG_PASSWORD).
|
||||
if [ -f "$SCRIPT_DIR/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "$SCRIPT_DIR/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
WORKING_DIR=$(pwd)
|
||||
|
||||
export RUST_LOG=trace
|
||||
@@ -19,6 +27,9 @@ export BAL_PUSHER_REGTEST_ZMQ_HASHBLOCK=tcp://127.0.0.1:21332
|
||||
export BAL_PUSHER_SEND_STATS=true
|
||||
export WELIST_SERVER_URL=http://localhost:8086
|
||||
export WELIST_SKIP_URL_VALIDATION=true
|
||||
# Sign WeTime pings with the v2 canonical JSON scheme (DNSSEC/onion public key
|
||||
# discovery). Remove this line to fall back to the legacy v1 signature.
|
||||
export WELIST_PING_SIG_VERSION=2
|
||||
export BAL_SERVER_URL="http://127.0.0.1:9133"
|
||||
export SSL_KEY_PATH="$WORKING_DIR/private_key.pem"
|
||||
|
||||
|
||||
45
bal-server-2.sh
Executable file
45
bal-server-2.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Second bal-server instance: own port, own database, same key pair.
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
. "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
# Load local, gitignored environment overrides (e.g. BAL_PG_PASSWORD).
|
||||
if [ -f "$SCRIPT_DIR/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "$SCRIPT_DIR/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
WORKING_DIR=$(pwd)
|
||||
if [ ! -f "$WORKING_DIR/public_key.pem" ]; then
|
||||
echo_i "creating keypairs"
|
||||
openssl genpkey -algorithm ED25519 -out private_key.pem
|
||||
openssl pkey -in private_key.pem -pubout -out public_key.pem
|
||||
fi
|
||||
|
||||
export RUST_LOG="trace"
|
||||
export BAL_SERVER_DB_FILE="$WORKING_DIR/bal2.db"
|
||||
export BAL_SERVER_INFO="BAL devel willexecutor server 2"
|
||||
export BAL_SERVER_BIND_ADDRESS="127.0.0.1"
|
||||
export BAL_SERVER_BIND_PORT=9134
|
||||
export BAL_SERVER_PUB_KEY_PATH="$WORKING_DIR/public_key.pem"
|
||||
export BAL_SERVER_EXPOSE_STATS=true
|
||||
|
||||
# Dedicated xpub from the regtest descriptor wallet "server2" (BIP84 account),
|
||||
# so that the two instances never derive the same deposit addresses.
|
||||
export BAL_SERVER_REGTEST_ADDRESS="tpubDCPHz2qeRgsqYMKkrEidz5NMFmCv36NX1C3FFYjg2hnvgrBTiw37uXxVnrvF6y6dswhJrQF9FTFW3jP334mMrXD5yndoHrNmjgToAaBP7iJ"
|
||||
export BAL_SERVER_REGTEST_FIXED_FEE=1000
|
||||
export BAL_SERVER_DB_BACKEND="postgresql"
|
||||
|
||||
# Instance-specific settings must win over .env, which defines BAL_PG_DB=baldb.
|
||||
export BAL_PG_DB="baldb2"
|
||||
unset BAL_SERVER_PG_DSN
|
||||
|
||||
if [ "$BAL_SERVER_DB_BACKEND" = "postgresql" ]; then
|
||||
. "$SCRIPT_DIR/pg-server.sh"
|
||||
fi
|
||||
|
||||
cargo run --bin=bal-server
|
||||
@@ -4,6 +4,14 @@ set -e
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
. "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
# Load local, gitignored environment overrides (e.g. BAL_PG_PASSWORD).
|
||||
if [ -f "$SCRIPT_DIR/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "$SCRIPT_DIR/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
WORKING_DIR=$(pwd)
|
||||
if [ ! -f "$WORKING_DIR/public_key.pem" ]; then
|
||||
echo_i "creating keypairs"
|
||||
|
||||
@@ -40,7 +40,7 @@ User
|
||||
4. **Monitoring**: The `bal-pusher` listens to the ZMQ `hashblock` topic with a 5-second receive timeout. When a new block is detected, it fetches blockchain info via RPC.
|
||||
5. **Evaluation**: The pusher queries the database for transactions with `status=0` and compares their locktime against the blockchain's best block height or median time (for timestamp-based locktimes above `LOCKTIME_THRESHOLD`).
|
||||
6. **Broadcast**: If the locktime is satisfied, the pusher sends the transaction via `sendrawtransaction` and updates the status to `1` (sent) or `2` (failed with error stored in `push_err`).
|
||||
7. **Statistics**: The pusher periodically calculates statistics and sends them to a remote `welist` server using an Ed25519-signed POST request. The server also exposes stats via the `GET /:network/stats` endpoint if `expose_stats` is enabled.
|
||||
7. **Statistics**: The pusher periodically calculates statistics and sends them to a remote `welist` server using an Ed25519-signed POST request. The signature scheme is selected by `WELIST_PING_SIG_VERSION`: legacy v1, or v2 over a canonical JSON payload whose `info`, `base_fee` and `version` come from the local `/{chain}/info` endpoint. The server also exposes stats via the `GET /:network/stats` endpoint if `expose_stats` is enabled.
|
||||
|
||||
## State Machine
|
||||
|
||||
|
||||
@@ -193,6 +193,23 @@ This is the async daemon that monitors the blockchain and pushes pending transac
|
||||
- Stats are inserted into `tbl_stats` with `ON CONFLICT(chain) DO UPDATE`.
|
||||
- The stats payload is signed with Ed25519 and POSTed to `{welist_url}/ping`.
|
||||
- The `WELIST_SERVER_URL` is validated via `is_valid_welist_url()` before sending (can be bypassed with `WELIST_SKIP_URL_VALIDATION=true`).
|
||||
- `WELIST_PING_SIG_VERSION` selects the signature scheme:
|
||||
- `1` (default): legacy concatenation of URL, chain, height, median time and
|
||||
block hash. The WeTime service verifies it against `/.pub_key.pem`.
|
||||
- `2`: canonical JSON signed with the `WELIST-PING-V2\n` prefix. The pusher
|
||||
fetches `info`, `base_fee` and `version` from the local
|
||||
`/{chain}/info` endpoint and signs them together with the block metadata.
|
||||
The WeTime service discovers the public key from a DNSSEC-validated
|
||||
`_welist-key.<host>` TXT record or from a v3 onion address, so no HTTP
|
||||
callback is needed. The `url` is trimmed of trailing slashes to match the
|
||||
service-side normalization; absent metadata is signed as `null`.
|
||||
- `bal-pusher.sh` sets `WELIST_PING_SIG_VERSION=2` for the development setup;
|
||||
remove that line to fall back to the legacy scheme.
|
||||
- To publish the v2 public key, derive the base64 SubjectPublicKeyInfo DER:
|
||||
`openssl pkey -in private_key.pem -pubout -outform DER | base64 -w0` and
|
||||
publish it in a DNSSEC-signed TXT record named `_welist-key.<host>` with value
|
||||
`v=we1; alg=ed25519; key=<base64>`. A v3 onion URL needs no DNS record: its
|
||||
Ed25519 identity key is embedded in the address.
|
||||
|
||||
### ZMQ Timeout Handling
|
||||
- Uses `set_rcvtimeo(5000)` (5-second timeout).
|
||||
@@ -208,5 +225,6 @@ All configuration is via environment variables (no config files):
|
||||
- `SSL_KEY_PATH`: Path to Ed25519 private key for signing stats.
|
||||
- `WELIST_SERVER_URL`: URL to POST stats to (validated against SSRF).
|
||||
- `WELIST_SKIP_URL_VALIDATION`: Bypass URL validation (for testing).
|
||||
- `WELIST_PING_SIG_VERSION`: WeTime ping signature scheme (`1` legacy, `2` canonical JSON).
|
||||
- `BAL_PUSHER_PREFER_IPV6`: Pin HTTP connection to IPv6 address.
|
||||
- Per-network: `BAL_PUSHER_{NETWORK}_HOST`, `_PORT`, `_DIR_PATH`, `_DB_FIELD`, `_COOKIE_FILE`, `_RPC_USER`, `_RPC_PASSWORD`, `_ZMQ_HASHBLOCK`.
|
||||
|
||||
@@ -147,11 +147,34 @@ The client tries username/password auth first, then falls back to cookie file au
|
||||
|
||||
---
|
||||
|
||||
## Ed25519 Stats Signing
|
||||
## Ed25519 WeTime Ping Signing
|
||||
|
||||
The pusher signs the statistics payload before sending it to the `welist` server:
|
||||
1. Collects statistics from the database.
|
||||
2. Serializes the stats as JSON.
|
||||
3. Signs the JSON payload with the Ed25519 private key (`privkey.pem`).
|
||||
4. Sends the payload with the base64-encoded signature in the `X-Signature` header.
|
||||
5. The `welist` server can verify the signature using the public key served at `GET /.pub_key.pem`.
|
||||
The pusher signs the report before sending it to the `welist` server. The
|
||||
scheme is selected with `WELIST_PING_SIG_VERSION`:
|
||||
|
||||
### Version 2 (canonical JSON, preferred)
|
||||
|
||||
1. Fetches `info`, `base_fee` and `version` from the local
|
||||
`GET /{chain}/info` endpoint (failure falls back to `null`).
|
||||
2. Builds a compact JSON object with lexicographically sorted keys, every field
|
||||
present, and `null` for absent optional values:
|
||||
`base_fee`, `chain`, `height`, `info`, `last_block_hash`, `median_time`,
|
||||
`sig_version`, `url`, `version`.
|
||||
3. Signs `WELIST-PING-V2\n` followed by that JSON with the Ed25519 private key
|
||||
(`SSL_KEY_PATH`).
|
||||
4. POSTs the same fields plus the base64 signature and `"sig_version": 2` to
|
||||
`{WELIST_SERVER_URL}/ping`.
|
||||
5. The `welist` server discovers the public key from a DNSSEC-validated
|
||||
`_welist-key.<host>` TXT record (`v=we1; alg=ed25519; key=<base64 SPKI DER>`)
|
||||
or from a v3 `.onion` address. No HTTP callback to `/.pub_key.pem` is made.
|
||||
|
||||
The `url` is trimmed of surrounding whitespace and trailing slashes, and the
|
||||
`chain` is lowercase, matching the service-side canonicalization.
|
||||
|
||||
### Version 1 (legacy)
|
||||
|
||||
1. Concatenates `url + chain + height + median_time + last_block_hash`.
|
||||
2. Signs that string with the Ed25519 private key.
|
||||
3. POSTs the fields plus the base64 signature to `{WELIST_SERVER_URL}/ping`.
|
||||
4. The `welist` server verifies the signature with the public key served at
|
||||
`GET /.pub_key.pem`; this scheme only refreshes `last_update`.
|
||||
|
||||
@@ -69,6 +69,7 @@ the handler (including request body reading) exceeds the limit.
|
||||
| `BAL_PUSHER_PREFER_IPV6` | `false` | Pin HTTP connection to first IPv6 address (for broken IPv4 routes) |
|
||||
| `WELIST_SERVER_URL` | `"https://welist.bitcoin-after.life"` | URL to POST signed stats to (validated against SSRF) |
|
||||
| `WELIST_SKIP_URL_VALIDATION` | `false` | Bypass SSRF URL validation (for testing only) |
|
||||
| `WELIST_PING_SIG_VERSION` | `1` | WeTime ping signature scheme: `1` (legacy) or `2` (canonical JSON) |
|
||||
|
||||
#### Per-Network Settings
|
||||
|
||||
@@ -200,6 +201,19 @@ export $(grep -v '^#' bal-server.env | xargs)
|
||||
RUST_LOG=info cargo run --bin=bal-server 2>&1
|
||||
```
|
||||
|
||||
### `pg-server.sh` (Local PostgreSQL)
|
||||
Starts the `bal-pg` Docker container used when `BAL_SERVER_DB_BACKEND=postgresql`.
|
||||
|
||||
- Requires `BAL_PG_PASSWORD` (loaded from `.env` by the startup scripts).
|
||||
- Publishes port `5432` on `127.0.0.1` only by default. Set
|
||||
`BAL_PG_BIND_ADDR=0.0.0.0` only if remote access is really required; the
|
||||
script prints a warning when the database is not bound to loopback.
|
||||
- Stores data in the Docker volume `BAL_PG_VOLUME` (default `bal-pg-data`), so
|
||||
the container can be recreated without losing data.
|
||||
- Fails if the existing container publishes the port on a different interface.
|
||||
Run `./pg-server.sh --recreate` to recreate it; the data volume is preserved
|
||||
(and migrated to `BAL_PG_VOLUME` if it changed).
|
||||
|
||||
### `bal-pusher.sh` (Development Pusher Startup)
|
||||
Sources `bal-pusher.env` and runs the pusher with a network argument:
|
||||
```bash
|
||||
@@ -282,6 +296,7 @@ Key points:
|
||||
- [ ] Database file owned by `bal` user (`chown bal:bal /var/bal/bal.db`).
|
||||
- [ ] Database file permissions `600` (`chmod 600 /var/bal/bal.db`).
|
||||
- [ ] `.env` files in `.gitignore` and not committed.
|
||||
- [ ] PostgreSQL (if used) published on `127.0.0.1` only (`BAL_PG_BIND_ADDR`).
|
||||
- [ ] `private_key.pem` / `privkey.pem` not in the repository.
|
||||
- [ ] `public_key.pem` readable by Nginx if served directly.
|
||||
|
||||
|
||||
97
pg-server.sh
97
pg-server.sh
@@ -5,15 +5,95 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
. "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
PG_CONTAINER="bal-pg"
|
||||
PG_IMAGE="${BAL_PG_IMAGE:-postgres:16-alpine}"
|
||||
PG_VOLUME="${BAL_PG_VOLUME:-bal-pg-data}"
|
||||
PG_USER="${BAL_PG_USER:-bal}"
|
||||
PG_DB="${BAL_PG_DB:-baldb}"
|
||||
# The password MUST come from the environment; never hardcode it.
|
||||
PG_PASSWORD="${BAL_PG_PASSWORD:?BAL_PG_PASSWORD must be set (see .env.example)}"
|
||||
PG_DSN="${BAL_SERVER_PG_DSN:-postgres://${PG_USER}:${PG_PASSWORD}@localhost:5432/${PG_DB}}"
|
||||
|
||||
# Publish the port on the loopback interface only by default. Set
|
||||
# BAL_PG_BIND_ADDR=0.0.0.0 to expose PostgreSQL on every interface (not
|
||||
# recommended: the database would be reachable from the network).
|
||||
PG_BIND_ADDR="${BAL_PG_BIND_ADDR:-127.0.0.1}"
|
||||
PG_PORT=5432
|
||||
|
||||
# Address the local clients use to reach the container. When the port is
|
||||
# published on all interfaces, loopback is still the safest connect address.
|
||||
case "$PG_BIND_ADDR" in
|
||||
0.0.0.0|::|"") PG_CONNECT_ADDR="127.0.0.1" ;;
|
||||
*) PG_CONNECT_ADDR="$PG_BIND_ADDR" ;;
|
||||
esac
|
||||
PG_DSN="${BAL_SERVER_PG_DSN:-postgres://${PG_USER}:${PG_PASSWORD}@${PG_CONNECT_ADDR}:${PG_PORT}/${PG_DB}}"
|
||||
|
||||
# Print the host IP the container port is currently published on.
|
||||
# An empty value means Docker publishes it on all interfaces.
|
||||
pg_container_bind_addr() {
|
||||
docker inspect --format '{{range $port, $bindings := .HostConfig.PortBindings}}{{range $bindings}}{{.HostIp}}{{end}}{{end}}' "$PG_CONTAINER" 2>/dev/null
|
||||
}
|
||||
|
||||
warn_if_public() {
|
||||
case "$PG_BIND_ADDR" in
|
||||
127.0.0.1|::1|localhost) ;;
|
||||
*)
|
||||
echo_w "WARNING: PostgreSQL is published on ${PG_BIND_ADDR}:${PG_PORT} and is reachable from the network."
|
||||
echo_w " Set BAL_PG_BIND_ADDR=127.0.0.1 unless remote access is really required."
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Fail closed when an existing container publishes the port elsewhere.
|
||||
# Docker cannot change port bindings of a running/stopped container.
|
||||
require_expected_bind_addr() {
|
||||
local current
|
||||
current=$(pg_container_bind_addr)
|
||||
[ "$current" = "$PG_BIND_ADDR" ] && return 0
|
||||
|
||||
echo_e "Container '$PG_CONTAINER' publishes port ${PG_PORT} on '${current:-0.0.0.0}', expected '$PG_BIND_ADDR'."
|
||||
echo_e "Docker cannot change port bindings of an existing container."
|
||||
echo_e "Recreate it while keeping its data with:"
|
||||
echo_e " $0 --recreate"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Recreate the container so that the new port binding takes effect.
|
||||
# The data volume is reused (or copied into BAL_PG_VOLUME when it changes).
|
||||
recreate_container() {
|
||||
local mount_type old_volume
|
||||
mount_type=$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql/data"}}{{.Type}}{{end}}{{end}}' "$PG_CONTAINER" 2>/dev/null || true)
|
||||
old_volume=$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql/data"}}{{.Name}}{{end}}{{end}}' "$PG_CONTAINER" 2>/dev/null || true)
|
||||
|
||||
if [ "$mount_type" = "bind" ]; then
|
||||
echo_e "Container '$PG_CONTAINER' uses a bind mount for its data; refusing to recreate it automatically."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo_i "Removing container '$PG_CONTAINER' (its data volume is preserved)..."
|
||||
docker rm -f "$PG_CONTAINER" >/dev/null
|
||||
|
||||
if [ -n "$old_volume" ] && [ "$old_volume" != "$PG_VOLUME" ]; then
|
||||
echo_i "Migrating data from volume '$old_volume' to '$PG_VOLUME'..."
|
||||
docker volume create "$PG_VOLUME" >/dev/null
|
||||
docker run --rm --entrypoint sh \
|
||||
-v "$old_volume:/from:ro" \
|
||||
-v "$PG_VOLUME:/to" \
|
||||
"$PG_IMAGE" -c 'cp -a /from/. /to/'
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--recreate" ]; then
|
||||
if docker ps -a --format '{{.Names}}' | grep -q "^${PG_CONTAINER}$"; then
|
||||
recreate_container
|
||||
fi
|
||||
fi
|
||||
|
||||
warn_if_public
|
||||
|
||||
if docker ps --format '{{.Names}}' | grep -q "^${PG_CONTAINER}$"; then
|
||||
echo_i "PostgreSQL container '$PG_CONTAINER' is already running"
|
||||
require_expected_bind_addr
|
||||
elif docker ps -a --format '{{.Names}}' | grep -q "^${PG_CONTAINER}$"; then
|
||||
require_expected_bind_addr
|
||||
echo_i "Starting existing PostgreSQL container '$PG_CONTAINER'..."
|
||||
docker start "$PG_CONTAINER" >/dev/null 2>&1
|
||||
else
|
||||
@@ -22,13 +102,24 @@ else
|
||||
-e POSTGRES_USER="$PG_USER" \
|
||||
-e POSTGRES_PASSWORD="$PG_PASSWORD" \
|
||||
-e POSTGRES_DB="$PG_DB" \
|
||||
-p 5432:5432 postgres:16-alpine
|
||||
-p "${PG_BIND_ADDR}:${PG_PORT}:5432" \
|
||||
-v "${PG_VOLUME}:/var/lib/postgresql/data" \
|
||||
"$PG_IMAGE" >/dev/null
|
||||
fi
|
||||
|
||||
echo_i "Waiting for PostgreSQL..."
|
||||
until docker exec "$PG_CONTAINER" pg_isready -U "$PG_USER" >/dev/null 2>&1; do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# The container is created with a single database, so create the requested one
|
||||
# if it does not exist yet (e.g. a second instance with its own database).
|
||||
if ! docker exec "$PG_CONTAINER" psql -U "$PG_USER" -d postgres -tAc \
|
||||
"SELECT 1 FROM pg_database WHERE datname = '$PG_DB'" | grep -q 1; then
|
||||
echo_i "Creating database '$PG_DB'..."
|
||||
docker exec "$PG_CONTAINER" createdb -U "$PG_USER" "$PG_DB"
|
||||
fi
|
||||
|
||||
# Do not echo the DSN: it embeds the password.
|
||||
echo_s "PostgreSQL ready (user=$PG_USER, db=$PG_DB, host=localhost:5432)"
|
||||
echo_s "PostgreSQL ready (user=$PG_USER, db=$PG_DB, host=${PG_CONNECT_ADDR}:${PG_PORT})"
|
||||
export BAL_SERVER_PG_DSN="$PG_DSN"
|
||||
|
||||
@@ -10,7 +10,7 @@ use log::{debug, error, info, trace, warn};
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::env;
|
||||
use std::error::Error as StdError;
|
||||
use std::str;
|
||||
@@ -27,6 +27,19 @@ use url::Url;
|
||||
// BIP-65: locktime values below this are block heights, at or above are UNIX timestamps.
|
||||
const LOCKTIME_THRESHOLD: i64 = 500_000_000;
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Legacy ping signature: concatenation of URL, chain, height, median time and
|
||||
/// block hash, verified against `/.pub_key.pem`.
|
||||
const PING_SIG_VERSION_LEGACY: u8 = 1;
|
||||
/// Canonical JSON ping signature (`WELIST-PING-V2\n` + sorted compact JSON).
|
||||
const PING_SIG_VERSION_CANONICAL: u8 = 2;
|
||||
/// Domain-separation prefix signed before a v2 canonical JSON payload.
|
||||
const PING_SIGNATURE_PREFIX: &[u8] = b"WELIST-PING-V2\n";
|
||||
/// Maximum accepted `info` length, matching the WeTime `/ping` validation.
|
||||
const MAX_INFO_BYTES: usize = 4096;
|
||||
/// Maximum accepted `version` length, matching the WeTime `/ping` validation.
|
||||
const MAX_VERSION_BYTES: usize = 64;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct MyConfig {
|
||||
db_backend: String,
|
||||
@@ -41,6 +54,9 @@ struct MyConfig {
|
||||
send_stats: bool,
|
||||
url: String,
|
||||
ssl_key_path: String,
|
||||
/// WeTime ping signature version. Defaults to the legacy scheme so an
|
||||
/// unmodified deployment keeps working.
|
||||
ping_sig_version: u8,
|
||||
}
|
||||
|
||||
impl Default for MyConfig {
|
||||
@@ -61,10 +77,42 @@ impl Default for MyConfig {
|
||||
.unwrap_or(false),
|
||||
url: env::var("BAL_SERVER_URL").unwrap_or("http://127.0.0.1:9133".to_string()),
|
||||
ssl_key_path: env::var("SSL_KEY_PATH").unwrap_or("privkey.pem".to_string()),
|
||||
ping_sig_version: parse_ping_sig_version(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `WELIST_PING_SIG_VERSION`, falling back to the legacy scheme.
|
||||
///
|
||||
/// Unsupported or malformed values are reported and treated as `1` so a typo
|
||||
/// cannot silently break stats reporting.
|
||||
fn parse_ping_sig_version() -> u8 {
|
||||
match env::var("WELIST_PING_SIG_VERSION") {
|
||||
Ok(value) => match value.trim().parse::<u8>() {
|
||||
Ok(version)
|
||||
if version == PING_SIG_VERSION_LEGACY || version == PING_SIG_VERSION_CANONICAL =>
|
||||
{
|
||||
version
|
||||
}
|
||||
Ok(version) => {
|
||||
warn!(
|
||||
"Unsupported WELIST_PING_SIG_VERSION={}, using legacy scheme",
|
||||
version
|
||||
);
|
||||
PING_SIG_VERSION_LEGACY
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"Invalid WELIST_PING_SIG_VERSION={:?} ({}), using legacy scheme",
|
||||
value, err
|
||||
);
|
||||
PING_SIG_VERSION_LEGACY
|
||||
}
|
||||
},
|
||||
Err(_) => PING_SIG_VERSION_LEGACY,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct NetworkParams {
|
||||
host: String,
|
||||
@@ -360,6 +408,141 @@ fn new_welist_client() -> rClient {
|
||||
.unwrap_or_else(|_| rClient::new())
|
||||
}
|
||||
|
||||
/// Metadata advertised by the local bal-server `/{chain}/info` endpoint.
|
||||
///
|
||||
/// These values are included in a v2 ping so the executor can publish signed
|
||||
/// `info`, `base_fee` and `version` values to the WeTime service.
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
struct ExecutorMetadata {
|
||||
#[serde(default)]
|
||||
info: Option<String>,
|
||||
#[serde(default)]
|
||||
base_fee: Option<i64>,
|
||||
#[serde(default)]
|
||||
version: Option<String>,
|
||||
}
|
||||
|
||||
/// Normalize an executor URL exactly like the WeTime `/ping` handler.
|
||||
fn normalize_executor_url(url: &str) -> String {
|
||||
url.trim().trim_end_matches('/').to_string()
|
||||
}
|
||||
|
||||
/// Drop metadata that the WeTime service would reject.
|
||||
fn sanitize_metadata(metadata: ExecutorMetadata) -> ExecutorMetadata {
|
||||
ExecutorMetadata {
|
||||
info: metadata
|
||||
.info
|
||||
.filter(|value| !value.is_empty() && value.len() <= MAX_INFO_BYTES),
|
||||
base_fee: metadata.base_fee.filter(|value| *value >= 0),
|
||||
version: metadata
|
||||
.version
|
||||
.filter(|value| !value.is_empty() && value.len() <= MAX_VERSION_BYTES),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return `true` when the URL host is a v3 `.onion` address.
|
||||
///
|
||||
/// The pusher cannot reach its own onion address without a Tor proxy, so v2
|
||||
/// pings for onion URLs are signed with `null` metadata instead.
|
||||
fn is_onion_url(url: &str) -> bool {
|
||||
Url::parse(url)
|
||||
.ok()
|
||||
.and_then(|parsed| {
|
||||
parsed
|
||||
.host_str()
|
||||
.map(|host| host.trim_end_matches('.').to_ascii_lowercase())
|
||||
})
|
||||
.is_some_and(|host| host.ends_with(".onion"))
|
||||
}
|
||||
|
||||
/// Fetch the signed metadata published by the local bal-server.
|
||||
///
|
||||
/// Failure is not fatal: a v2 ping is still valid with `null` metadata, it just
|
||||
/// cannot refresh those columns on the WeTime service.
|
||||
async fn fetch_executor_metadata(cfg: &MyConfig, chain: &str) -> ExecutorMetadata {
|
||||
let base_url = normalize_executor_url(&cfg.url);
|
||||
if is_onion_url(&base_url) {
|
||||
debug!("skipping executor metadata fetch for onion URL");
|
||||
return ExecutorMetadata::default();
|
||||
}
|
||||
let endpoint = format!("{}/{}/info", base_url, chain);
|
||||
let client = new_welist_client();
|
||||
match client.get(&endpoint).send().await {
|
||||
Ok(response) if response.status().is_success() => {
|
||||
match response.json::<ExecutorMetadata>().await {
|
||||
Ok(metadata) => {
|
||||
debug!("executor metadata: {:?}", metadata);
|
||||
sanitize_metadata(metadata)
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("cannot parse executor metadata from {}: {}", endpoint, err);
|
||||
ExecutorMetadata::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response) => {
|
||||
warn!(
|
||||
"executor metadata endpoint {} returned {}",
|
||||
endpoint,
|
||||
response.status()
|
||||
);
|
||||
ExecutorMetadata::default()
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("cannot fetch executor metadata from {}: {}", endpoint, err);
|
||||
ExecutorMetadata::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fields covered by a v2 WeTime ping signature.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct PingPayload<'a> {
|
||||
/// Normalized executor URL.
|
||||
url: &'a str,
|
||||
/// Canonical lowercase chain name.
|
||||
chain: &'a str,
|
||||
/// Self-reported block height.
|
||||
height: u64,
|
||||
/// Self-reported median time.
|
||||
median_time: u64,
|
||||
/// Self-reported block hash.
|
||||
last_block_hash: &'a str,
|
||||
/// Optional executor description.
|
||||
info: Option<&'a str>,
|
||||
/// Optional executor base fee.
|
||||
base_fee: Option<i64>,
|
||||
/// Optional executor version.
|
||||
version: Option<&'a str>,
|
||||
}
|
||||
|
||||
impl PingPayload<'_> {
|
||||
/// Build the exact bytes covered by a v2 ping signature.
|
||||
///
|
||||
/// The output is `WELIST-PING-V2\n` followed by compact JSON with sorted
|
||||
/// keys, every field present, and `null` for absent optional values. This
|
||||
/// must match the canonicalization performed by the WeTime `/ping` handler
|
||||
/// byte for byte.
|
||||
fn canonical_bytes(&self) -> Result<Vec<u8>, serde_json::Error> {
|
||||
let mut fields: BTreeMap<&str, serde_json::Value> = BTreeMap::new();
|
||||
fields.insert("base_fee", json!(self.base_fee));
|
||||
fields.insert("chain", json!(self.chain));
|
||||
fields.insert("height", json!(self.height));
|
||||
fields.insert("info", json!(self.info));
|
||||
fields.insert("last_block_hash", json!(self.last_block_hash));
|
||||
fields.insert("median_time", json!(self.median_time));
|
||||
fields.insert("sig_version", json!(PING_SIG_VERSION_CANONICAL));
|
||||
fields.insert("url", json!(self.url));
|
||||
fields.insert("version", json!(self.version));
|
||||
|
||||
let body = serde_json::to_vec(&fields)?;
|
||||
let mut message = Vec::with_capacity(PING_SIGNATURE_PREFIX.len() + body.len());
|
||||
message.extend_from_slice(PING_SIGNATURE_PREFIX);
|
||||
message.extend_from_slice(&body);
|
||||
Ok(message)
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_stats_report(
|
||||
cfg: &MyConfig,
|
||||
bcinfo: GetBlockchainInfoResult,
|
||||
@@ -383,32 +566,80 @@ async fn send_stats_report(
|
||||
let url = format!("{}/ping", welist_url);
|
||||
debug!("welist url: {}", url);
|
||||
let chain = bcinfo.chain.to_string().to_lowercase();
|
||||
let message = format!(
|
||||
"{0}{1}{2}{3}{4}",
|
||||
cfg.url, chain, bcinfo.blocks, bcinfo.median_time, bcinfo.best_block_hash
|
||||
);
|
||||
trace!("message to be sent: {}", message);
|
||||
let sign = match sign_message(cfg.ssl_key_path.as_str(), message.as_str()) {
|
||||
Ok(signature) => signature,
|
||||
Err(e) => {
|
||||
error!("Cannot sign stats report, skipping: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let response = client
|
||||
let last_block_hash = bcinfo.best_block_hash.to_string();
|
||||
|
||||
let request = client
|
||||
.post(url)
|
||||
.header("User-Agent", format!("bal-pusher/{}", VERSION))
|
||||
.json(&json!(
|
||||
.header("User-Agent", format!("bal-pusher/{}", VERSION));
|
||||
|
||||
let request = if cfg.ping_sig_version == PING_SIG_VERSION_CANONICAL {
|
||||
let executor_url = normalize_executor_url(&cfg.url);
|
||||
let metadata = fetch_executor_metadata(cfg, &chain).await;
|
||||
let payload = PingPayload {
|
||||
url: &executor_url,
|
||||
chain: &chain,
|
||||
height: bcinfo.blocks,
|
||||
median_time: bcinfo.median_time,
|
||||
last_block_hash: &last_block_hash,
|
||||
info: metadata.info.as_deref(),
|
||||
base_fee: metadata.base_fee,
|
||||
version: metadata.version.as_deref(),
|
||||
};
|
||||
let message = match payload.canonical_bytes() {
|
||||
Ok(message) => message,
|
||||
Err(err) => {
|
||||
error!("Cannot build canonical ping payload, skipping: {}", err);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
trace!(
|
||||
"canonical message to be sent: {}",
|
||||
String::from_utf8_lossy(&message)
|
||||
);
|
||||
let sign = match sign_bytes(cfg.ssl_key_path.as_str(), &message) {
|
||||
Ok(signature) => signature,
|
||||
Err(e) => {
|
||||
error!("Cannot sign stats report, skipping: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
request.json(&json!({
|
||||
"url": executor_url,
|
||||
"chain": chain,
|
||||
"height": bcinfo.blocks,
|
||||
"median_time": bcinfo.median_time,
|
||||
"last_block_hash": last_block_hash,
|
||||
"signature": sign,
|
||||
"sig_version": PING_SIG_VERSION_CANONICAL,
|
||||
"info": metadata.info,
|
||||
"base_fee": metadata.base_fee,
|
||||
"version": metadata.version,
|
||||
}))
|
||||
} else {
|
||||
let message = format!(
|
||||
"{0}{1}{2}{3}{4}",
|
||||
cfg.url, chain, bcinfo.blocks, bcinfo.median_time, last_block_hash
|
||||
);
|
||||
trace!("message to be sent: {}", message);
|
||||
let sign = match sign_message(cfg.ssl_key_path.as_str(), message.as_str()) {
|
||||
Ok(signature) => signature,
|
||||
Err(e) => {
|
||||
error!("Cannot sign stats report, skipping: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
request.json(&json!(
|
||||
{
|
||||
"url": cfg.url,
|
||||
"chain": chain,
|
||||
"height": bcinfo.blocks,
|
||||
"median_time": bcinfo.median_time,
|
||||
"last_block_hash": bcinfo.best_block_hash,
|
||||
"last_block_hash": last_block_hash,
|
||||
"signature": sign,
|
||||
}))
|
||||
.send()
|
||||
.await?;
|
||||
};
|
||||
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
info!(
|
||||
@@ -420,14 +651,19 @@ async fn send_stats_report(
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn sign_message(private_key_path: &str, message: &str) -> Result<String, Box<dyn StdError>> {
|
||||
|
||||
fn sign_bytes(private_key_path: &str, message: &[u8]) -> Result<String, Box<dyn StdError>> {
|
||||
let signing_key = SigningKey::read_pkcs8_pem_file(private_key_path)
|
||||
.map_err(|e| format!("failed to parse private key PEM at {private_key_path}: {e}"))?;
|
||||
let signature = signing_key.sign(message.as_bytes());
|
||||
let signature = signing_key.sign(message);
|
||||
|
||||
Ok(general_purpose::STANDARD.encode(signature.to_bytes()))
|
||||
}
|
||||
|
||||
fn sign_message(private_key_path: &str, message: &str) -> Result<String, Box<dyn StdError>> {
|
||||
sign_bytes(private_key_path, message.as_bytes())
|
||||
}
|
||||
|
||||
fn parse_env(cfg: &mut MyConfig) {
|
||||
if let Ok(value) = env::var("BAL_SERVER_DB_BACKEND") {
|
||||
cfg.db_backend = value;
|
||||
@@ -624,4 +860,84 @@ mod tests {
|
||||
fn parse_host_port_invalid_url() {
|
||||
assert_eq!(parse_host_port("not a url"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_executor_url_trims_and_strips_trailing_slashes() {
|
||||
assert_eq!(
|
||||
normalize_executor_url(" https://we.example.com/ "),
|
||||
"https://we.example.com"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_executor_url("https://we.example.com"),
|
||||
"https://we.example.com"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn onion_urls_skip_metadata_fetch() {
|
||||
assert!(is_onion_url(
|
||||
"http://abcdefghijklmnopqrstuvwxyz234567abcdefghijklmnopqrstuv.onion"
|
||||
));
|
||||
assert!(!is_onion_url("https://we.example.com"));
|
||||
assert!(!is_onion_url("not a url"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_ping_payload_is_sorted_compact_and_complete() {
|
||||
let payload = PingPayload {
|
||||
url: "https://example.com",
|
||||
chain: "bitcoin",
|
||||
height: 100,
|
||||
median_time: 1000,
|
||||
last_block_hash: "abc",
|
||||
info: None,
|
||||
base_fee: None,
|
||||
version: None,
|
||||
};
|
||||
let payload = payload.canonical_bytes().unwrap();
|
||||
assert_eq!(
|
||||
String::from_utf8(payload).unwrap(),
|
||||
"WELIST-PING-V2\n{\"base_fee\":null,\"chain\":\"bitcoin\",\"height\":100,\"info\":null,\"last_block_hash\":\"abc\",\"median_time\":1000,\"sig_version\":2,\"url\":\"https://example.com\",\"version\":null}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_ping_payload_includes_metadata_and_escapes_strings() {
|
||||
let payload = PingPayload {
|
||||
url: "https://example.com",
|
||||
chain: "bitcoin",
|
||||
height: 1,
|
||||
median_time: 2,
|
||||
last_block_hash: "abc",
|
||||
info: Some("line\n\"quoted\""),
|
||||
base_fee: Some(1500),
|
||||
version: Some("0.3.0"),
|
||||
};
|
||||
let payload = payload.canonical_bytes().unwrap();
|
||||
let message = String::from_utf8(payload).unwrap();
|
||||
assert!(message.contains("\"base_fee\":1500"));
|
||||
assert!(message.contains("\"info\":\"line\\n\\\"quoted\\\"\""));
|
||||
assert!(message.contains("\"version\":\"0.3.0\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_metadata_drops_values_the_service_rejects() {
|
||||
let metadata = sanitize_metadata(ExecutorMetadata {
|
||||
info: Some("x".repeat(MAX_INFO_BYTES + 1)),
|
||||
base_fee: Some(-1),
|
||||
version: Some(String::new()),
|
||||
});
|
||||
assert_eq!(metadata.info, None);
|
||||
assert_eq!(metadata.base_fee, None);
|
||||
assert_eq!(metadata.version, None);
|
||||
|
||||
let metadata = sanitize_metadata(ExecutorMetadata {
|
||||
info: Some("ok".to_string()),
|
||||
base_fee: Some(0),
|
||||
version: Some(VERSION.to_string()),
|
||||
});
|
||||
assert_eq!(metadata.info.as_deref(), Some("ok"));
|
||||
assert_eq!(metadata.base_fee, Some(0));
|
||||
assert_eq!(metadata.version.as_deref(), Some(VERSION));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user