feat(net): parallel Check + timeout counters + GUI tooltips/order

Networking (anti-freeze):
- Add check_transactions_parallel: pressing "Check" now contacts will-executors
  concurrently with a fast-fail timeout (CHECK_TIMEOUT=8, 1 retry) and a global
  deadline (CHECK_GLOBAL_DEADLINE=30), so a single dead server no longer freezes
  the "checking transaction" dialog for ~140s (old default 10s x 10 retries).
- check_transaction now accepts timeout/max_retries/retry_sleep kwargs.
- Rewrite BalWindow.check_transactions_task to use the parallel helper with live
  progress + an elapsed-time counter "Checking transactions: 2/5 (4s / 30s)".

Reliable elapsed-time counters (Xs / DEADLINEs):
- Replace the unreliable raw heartbeat thread (whose pyqtSignal emissions were
  not marshalled and never repainted) with an on_tick callback driven from the
  CALLING thread in push/ping/check parallel helpers.
- Show the maximum wait too (e.g. "3s / 30s") so the user knows when the
  operation will give up, in the wizard Broadcasting, Ping and Check dialogs.
- Expose networking constants (PUSH_*/CHECK_*/DEFAULT_TIMEOUT) as Willexecutors
  class attributes for a single source of truth in the GUI.

GUI:
- Add hover tooltips: Wizard ("Wizard - Build your will"), Delivery time (truck),
  Check Alive (siren), Calendar, Check (refresh).
- Reorder the Will toolbar to: Wizard | Delivery time | Check Alive | Calendar |
  Check; tighten layout margins so it all fits the Will window.

Tests:
- parallel_ping_test.py: add coverage for check_transactions_parallel (parallel
  timing, global deadline, on_tick from the calling thread) and static checks
  that check_transactions_task/loop_push use the parallel helpers + on_tick
  counter with the 'Xs / Ns' format.

Verified: ruff (0 new issues), 182 official tests pass, parallel/smoke/gui_fixes/
windows_overflow/external_zip tests pass.
This commit is contained in:
GenSpark AI Developer
2026-06-15 18:37:24 +00:00
parent 42fc80bb55
commit 4abd2e508f
6 changed files with 689 additions and 58 deletions

View File

@@ -25,7 +25,36 @@ from electrum.network import Network
from .plugin_base import BalPlugin
# Per-request timeout (seconds) for interactive operations (ping / info /
# list download). These fail fast (no retries) so a dead server does not
# block the UI.
DEFAULT_TIMEOUT = 5
# Broadcast (pushtxs) timeouts. Broadcasting a will is important, so we keep a
# couple of quick retries to survive a transient hiccup -- but far from the old
# 10s x 10 retries + 30s sleeps (~140s) that froze the wizard on a dead server.
# Worst case per server is now ~ PUSH_TIMEOUT * (1 + PUSH_MAX_RETRIES)
# + PUSH_RETRY_SLEEP * PUSH_MAX_RETRIES = 8 * 3 + 1 * 2 = ~26s, and the wizard
# also enforces a global deadline on top of this (see push_transactions_parallel).
PUSH_TIMEOUT = 8
PUSH_MAX_RETRIES = 2
PUSH_RETRY_SLEEP = 1
# Global wall-clock deadline (seconds) for the whole parallel broadcast. Once
# it elapses we stop waiting for the still-pending servers, mark them as
# "Timeout" and let the wizard proceed instead of appearing stuck.
PUSH_GLOBAL_DEADLINE = 30
# Check (searchtx) timeouts. Used when the user presses "Check" to verify that
# each will-executor still holds the transaction. Like the broadcast path, the
# old defaults (10s x 10 retries + 30s sleeps ~= 140s per server) froze the
# "checking transaction" dialog on a single dead server. Fail fast with one
# quick retry, and cap the whole batch with a global deadline.
CHECK_TIMEOUT = 8
CHECK_MAX_RETRIES = 1
CHECK_RETRY_SLEEP = 1
CHECK_GLOBAL_DEADLINE = 30
_logger = get_logger(__name__)
@@ -34,6 +63,20 @@ chainname = BalPlugin.chainname
class Willexecutors:
# Expose the networking constants as class attributes so the GUI layer can
# reference them (e.g. to show the "Xs / DEADLINEs" countdown) without
# importing module-level names. Single source of truth: the module
# constants defined above.
DEFAULT_TIMEOUT = DEFAULT_TIMEOUT
PUSH_TIMEOUT = PUSH_TIMEOUT
PUSH_MAX_RETRIES = PUSH_MAX_RETRIES
PUSH_RETRY_SLEEP = PUSH_RETRY_SLEEP
PUSH_GLOBAL_DEADLINE = PUSH_GLOBAL_DEADLINE
CHECK_TIMEOUT = CHECK_TIMEOUT
CHECK_MAX_RETRIES = CHECK_MAX_RETRIES
CHECK_RETRY_SLEEP = CHECK_RETRY_SLEEP
CHECK_GLOBAL_DEADLINE = CHECK_GLOBAL_DEADLINE
@staticmethod
def save(bal_plugin, willexecutors):
_logger.debug(f"save {willexecutors},{chainname}")
@@ -241,7 +284,15 @@ class Willexecutors:
pass
@staticmethod
def push_transactions_to_willexecutor(willexecutor):
def push_transactions_to_willexecutor(
willexecutor, *, timeout=PUSH_TIMEOUT, max_retries=PUSH_MAX_RETRIES,
retry_sleep=PUSH_RETRY_SLEEP,
):
# ``timeout`` / ``max_retries`` / ``retry_sleep`` are forwarded to
# send_request so the broadcast fails fast on a dead/slow server instead
# of hanging for ~140s (the old default was 10s timeout x 10 retries +
# 30s of sleeps). A small number of quick retries still protects
# against a transient hiccup without freezing the wizard.
out = True
try:
_logger.debug(f"{willexecutor['url']}: {willexecutor['txs']}")
@@ -249,6 +300,9 @@ class Willexecutors:
"post",
willexecutor["url"] + "/" + chainname + "/pushtxs",
data=willexecutor["txs"].encode("ascii"),
timeout=timeout,
max_retries=max_retries,
retry_sleep=retry_sleep,
):
willexecutor["broadcast_status"] = _("Success")
_logger.debug(f"pushed: {w}")
@@ -304,7 +358,8 @@ class Willexecutors:
@staticmethod
def ping_servers_parallel(willexecutors, *, on_each=None, max_workers=8,
timeout=DEFAULT_TIMEOUT):
timeout=DEFAULT_TIMEOUT, on_tick=None,
tick_interval=1.0):
"""Ping every will-executor concurrently and report results as they
arrive.
@@ -324,10 +379,16 @@ class Willexecutors:
max_workers: maximum number of concurrent pings.
timeout: per-request timeout in seconds (fast-fail, no retries).
on_tick: optional ``callback()`` invoked periodically (every
``tick_interval`` seconds) **from the calling thread** while
waiting for servers, so a Qt caller can refresh an elapsed-time
counter from the same thread that drives ``on_each``.
Returns:
The same ``willexecutors`` mapping, updated in place.
"""
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import ThreadPoolExecutor, wait
from concurrent.futures import FIRST_COMPLETED
items = list(willexecutors.items())
if not items:
@@ -340,38 +401,75 @@ class Willexecutors:
ok = we.get("status") == 200
return url, we, ok
workers = max(1, min(max_workers, len(items)))
with ThreadPoolExecutor(max_workers=workers,
thread_name_prefix="bal-ping") as pool:
futures = [pool.submit(_ping_one, url, we) for url, we in items]
for fut in as_completed(futures):
def _fire_tick():
if on_tick is not None:
try:
url, we, ok = fut.result()
except Exception as e: # defensive: never let one server crash all
_logger.error(f"ping_servers_parallel worker error: {e}")
continue
willexecutors[url] = we
if on_each is not None:
on_tick()
except Exception as cb_err:
_logger.error(f"ping on_tick callback error: {cb_err}")
workers = max(1, min(max_workers, len(items)))
# Manual pool (no ``with``) so we can poll futures in short slices and
# drive ``on_tick`` from THIS thread between waits (reliable Qt repaint).
pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="bal-ping")
futures = {pool.submit(_ping_one, url, we) for url, we in items}
try:
pending = set(futures)
while pending:
done, pending = wait(
pending, timeout=tick_interval, return_when=FIRST_COMPLETED
)
for fut in done:
try:
on_each(url, we, ok)
except Exception as cb_err:
_logger.error(f"ping on_each callback error: {cb_err}")
url, we, ok = fut.result()
except Exception as e: # defensive: one server never crashes all
_logger.error(f"ping_servers_parallel worker error: {e}")
continue
willexecutors[url] = we
if on_each is not None:
try:
on_each(url, we, ok)
except Exception as cb_err:
_logger.error(f"ping on_each callback error: {cb_err}")
# Drive the elapsed-time counter from the calling thread.
_fire_tick()
finally:
try:
pool.shutdown(wait=False, cancel_futures=True)
except TypeError:
pool.shutdown(wait=False)
return willexecutors
@staticmethod
def push_transactions_parallel(willexecutors, *, on_each=None, max_workers=8):
def push_transactions_parallel(willexecutors, *, on_each=None, max_workers=8,
deadline=PUSH_GLOBAL_DEADLINE, on_timeout=None,
on_tick=None, tick_interval=1.0):
"""Push transactions to multiple will-executors concurrently.
Like :meth:`ping_servers_parallel` but for the ``pushtxs`` operation.
Each server keeps the historical retry behaviour of
:meth:`push_transactions_to_willexecutor` (which is important so a real
transaction is not lost to a transient hiccup), but the servers are now
contacted in parallel instead of one-after-another, and results are
reported via ``on_each(url, we_dict, ok, exc)`` as they complete.
Each server keeps a short retry behaviour
(:meth:`push_transactions_to_willexecutor`) so a real transaction is not
lost to a transient hiccup, but servers are contacted in parallel and
results are reported via ``on_each(url, we_dict, ok, exc)`` as they
complete.
Returns ``{url: (ok, exception_or_None)}``.
A global wall-clock ``deadline`` (seconds) caps the whole operation: if
some servers are still pending when it elapses, we stop waiting, mark
them via ``on_timeout(url, we_dict)`` and return, so the caller (the
wizard) is never stuck behind one unresponsive server. Pass
``deadline=None`` to wait indefinitely (old behaviour).
``on_tick()`` is invoked periodically (every ``tick_interval`` seconds)
**from the calling thread** while waiting for workers. This lets a Qt
caller refresh an elapsed-time counter from the same thread that drives
``on_each`` (so its pyqtSignal repaints reliably), instead of relying on
a separate heartbeat thread whose signal emissions are not marshalled.
Returns ``{url: (ok, exception_or_None)}`` for the servers that
answered in time (timed-out servers are reported via ``on_timeout``).
"""
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import ThreadPoolExecutor, wait
from concurrent.futures import FIRST_COMPLETED
targets = [(url, we) for url, we in willexecutors.items() if "txs" in we]
results = {}
@@ -387,22 +485,188 @@ class Willexecutors:
except Exception as e:
return url, we, False, e
workers = max(1, min(max_workers, len(targets)))
with ThreadPoolExecutor(max_workers=workers,
thread_name_prefix="bal-push") as pool:
futures = [pool.submit(_push_one, url, we) for url, we in targets]
for fut in as_completed(futures):
def _fire_tick():
if on_tick is not None:
try:
url, we, ok, exc = fut.result()
except Exception as e:
_logger.error(f"push_transactions_parallel worker error: {e}")
continue
results[url] = (ok, exc)
if on_each is not None:
on_tick()
except Exception as cb_err:
_logger.error(f"push on_tick callback error: {cb_err}")
workers = max(1, min(max_workers, len(targets)))
# NOTE: we do not use ``with ThreadPoolExecutor(...)`` here because its
# __exit__ calls shutdown(wait=True), which would block on a hung worker
# and defeat the whole point of the global deadline. We shut the pool
# down without waiting once the deadline elapses; the daemon worker(s)
# stuck on a dead socket will be torn down when their request finally
# times out (PUSH_TIMEOUT), without holding up the wizard.
pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="bal-push")
fut_to_url = {pool.submit(_push_one, url, we): (url, we)
for url, we in targets}
start = time.time()
try:
# Poll the futures in short slices so we can call ``on_tick`` from
# THIS thread between waits. ``wait(..., timeout=tick_interval)``
# returns as soon as a future completes OR the slice elapses,
# whichever comes first, so the counter advances ~once per second
# while the parallel push runs.
pending = set(fut_to_url.keys())
while pending:
if deadline is not None and (time.time() - start) >= deadline:
break
slice_timeout = tick_interval
if deadline is not None:
remaining = deadline - (time.time() - start)
slice_timeout = max(0.0, min(tick_interval, remaining))
done, pending = wait(
pending, timeout=slice_timeout, return_when=FIRST_COMPLETED
)
for fut in done:
try:
on_each(url, we, ok, exc)
except Exception as cb_err:
_logger.error(f"push on_each callback error: {cb_err}")
url, we, ok, exc = fut.result()
except Exception as e:
_logger.error(
f"push_transactions_parallel worker error: {e}"
)
continue
results[url] = (ok, exc)
if on_each is not None:
try:
on_each(url, we, ok, exc)
except Exception as cb_err:
_logger.error(f"push on_each callback error: {cb_err}")
# Drive the elapsed-time counter from the calling thread.
_fire_tick()
# Any server still pending here hit the global deadline.
if pending:
elapsed = time.time() - start
_logger.warning(
f"push global deadline ({deadline}s) reached after "
f"{elapsed:.1f}s; {len(pending)} server(s) "
f"did not answer in time"
)
for fut in pending:
url, we = fut_to_url[fut]
if url in results:
continue
if on_timeout is not None:
try:
on_timeout(url, we)
except Exception as cb_err:
_logger.error(
f"push on_timeout callback error: {cb_err}"
)
finally:
# Do not block on still-running workers (Python 3.9+: cancel queued).
try:
pool.shutdown(wait=False, cancel_futures=True)
except TypeError:
pool.shutdown(wait=False)
return results
@staticmethod
def check_transactions_parallel(items, *, on_each=None, max_workers=8,
deadline=CHECK_GLOBAL_DEADLINE,
on_timeout=None, on_tick=None,
tick_interval=1.0):
"""Check (searchtx) several will-executors concurrently.
Same design as :meth:`push_transactions_parallel`, but for the "Check"
operation: it verifies that each will-executor still holds its
transaction. ``items`` is an iterable of ``(wid, url)`` pairs (one per
will-item that has a will-executor).
Each server is contacted in parallel with a short fail-fast retry
(:meth:`check_transaction`), results are reported via
``on_each(wid, url, result_or_None, exc)`` as they arrive, ``on_tick()``
is called periodically from the calling thread to refresh a counter, and
a global ``deadline`` guarantees the dialog never freezes behind one
unresponsive server (pending servers are reported via
``on_timeout(wid, url)``).
Returns ``{wid: (result_or_None, exception_or_None)}`` for the servers
that answered in time.
"""
from concurrent.futures import ThreadPoolExecutor, wait
from concurrent.futures import FIRST_COMPLETED
targets = [(wid, url) for wid, url in items if url]
results = {}
if not targets:
return results
def _check_one(wid, url):
try:
res = Willexecutors.check_transaction(wid, url)
return wid, url, res, None
except Exception as e:
return wid, url, None, e
def _fire_tick():
if on_tick is not None:
try:
on_tick()
except Exception as cb_err:
_logger.error(f"check on_tick callback error: {cb_err}")
workers = max(1, min(max_workers, len(targets)))
# Manual pool (no ``with``): we must not block on a hung worker when the
# global deadline elapses (see push_transactions_parallel for details).
pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="bal-check")
fut_to_target = {pool.submit(_check_one, wid, url): (wid, url)
for wid, url in targets}
start = time.time()
try:
pending = set(fut_to_target.keys())
while pending:
if deadline is not None and (time.time() - start) >= deadline:
break
slice_timeout = tick_interval
if deadline is not None:
remaining = deadline - (time.time() - start)
slice_timeout = max(0.0, min(tick_interval, remaining))
done, pending = wait(
pending, timeout=slice_timeout, return_when=FIRST_COMPLETED
)
for fut in done:
try:
wid, url, res, exc = fut.result()
except Exception as e:
_logger.error(
f"check_transactions_parallel worker error: {e}"
)
continue
results[wid] = (res, exc)
if on_each is not None:
try:
on_each(wid, url, res, exc)
except Exception as cb_err:
_logger.error(f"check on_each callback error: {cb_err}")
# Drive the elapsed-time counter from the calling thread.
_fire_tick()
# Any server still pending here hit the global deadline.
if pending:
elapsed = time.time() - start
_logger.warning(
f"check global deadline ({deadline}s) reached after "
f"{elapsed:.1f}s; {len(pending)} server(s) "
f"did not answer in time"
)
for fut in pending:
wid, url = fut_to_target[fut]
if wid in results:
continue
if on_timeout is not None:
try:
on_timeout(wid, url)
except Exception as cb_err:
_logger.error(
f"check on_timeout callback error: {cb_err}"
)
finally:
try:
pool.shutdown(wait=False, cancel_futures=True)
except TypeError:
pool.shutdown(wait=False)
return results
@staticmethod
@@ -457,11 +721,14 @@ class Willexecutors:
return {}
@staticmethod
def check_transaction(txid, url):
def check_transaction(txid, url, *, timeout=CHECK_TIMEOUT,
max_retries=CHECK_MAX_RETRIES,
retry_sleep=CHECK_RETRY_SLEEP):
_logger.debug(f"{url}:{txid}")
try:
w = Willexecutors.send_request(
"post", url + "/searchtx", data=txid.encode("ascii")
"post", url + "/searchtx", data=txid.encode("ascii"),
timeout=timeout, max_retries=max_retries, retry_sleep=retry_sleep,
)
return w
except Exception as e: