"""
pool_worker.py

Abstract base class for all pool-registered workers.

Subclasses implement exactly one method:
    Execute(task_uuid: str) -> bool

Everything else is provided:
  - Heartbeat thread: Register(endpoint, state, pool_id) on tight interval
  - HTTP server: /health  /execute
  - Completion callback ("ping me back when you're done" — the event bus, operator 2026-07-11):
    if a task atom carries a `callback` descriptor {url, event, context} (or the legacy
    `callback_url` shorthand), the framework POSTs that completion event to that endpoint after
    Execute(), echoing `context` verbatim. Works for EVERY descendant — no per-worker code. A
    worker-level `default_callback` ctor arg supplies a fallback endpoint (per-task wins).
  - State machine: FREE | BUSY | FINISHING_AND_SHUTTING_DOWN | FROZEN
                   (8-bit ints, Lock-protected). The pool-service also
                   tracks a 4th state TASK_PICKED (int=1, lease marker
                   set on the dispatch path) — workers never write that
                   state; their next heartbeat overwrites it. FROZEN
                   (int=4, session-58) is a reversible HOLD: registered +
                   alive + heartbeating + READY but temporarily refusing
                   tasks (model stays resident). Polled via the per-worker
                   pipeline:freeze:{endpoint} flag; thaw = delete the key.
  - Signal handling: SIGTERM/SIGINT → drain, SIGHUP → reload, SIGUSR1 → GC, SIGUSR2 → dump
  - GC discipline: gc.collect(2) after every task before FREE transition
  - IP autodiscovery: WORKER_ENDPOINT > WORKER_ENDPOINT_IP > gethostbyname(gethostname())

State transition contract (enforced in /execute handler):
    _set_state(BUSY)          ← FIRST, unconditionally
    try:
        ok = Execute(task_uuid)
    except:
        _set_state(FREE)      ← rollback
        raise
    on_cleanup()              ← implementor releases task-local resources
    gc.collect(2)             ← forced full collection
    _set_state(FREE)          ← only now declare available

Environment variables (defaults when constructor args are None):
    POOL_SERVICE_URL          http://10.121.x.x:8780
    WORKER_POOL_ID            integer, which pool this worker belongs to
    WORKER_LISTEN_PORT        HTTP listen port (default 8800)
    WORKER_ENDPOINT           full override: ip:port
    WORKER_ENDPOINT_IP        IP override only
    WORKER_CONSUMER           consumer name for syslog / identification
    PS_PINGBACK_TIMEOUT       seconds (default 30)
    PS_HEARTBEAT_INTERVAL     seconds (default 25); must be < pingback_timeout
    REDIS_URL                 for task atom access

Constructor args (session-37 — added for multi-pool-per-process):
    pool_id                 int   override WORKER_POOL_ID
    listen_port             int   override WORKER_LISTEN_PORT
    consumer                str   override WORKER_CONSUMER
    install_signal_handlers bool  default True; set False when embedding multiple
                                  PoolWorker instances in one process (signals are
                                  process-scoped — last handler wins; let the
                                  hosting process install its own drain orchestration).

Single-worker callers (chunker, extractor, tagger, tokenizer, dissector,
storage, compressor): leave constructor empty — env defaults apply, behavior
preserved bit-for-bit.

Multi-worker callers (qwen_service: pool_id=2 forward_pass + pool_id=8 embed):
pass explicit pool_id/listen_port/consumer per instance + install_signal_handlers=False
on every instance; let main.py wire its own drain.
"""
from __future__ import annotations

import gc
import http.server
import json
import logging
import os
import signal
import socket
import sys
import syslog
import threading
import time
import traceback
import urllib.error
import urllib.request
from abc import ABC, abstractmethod

import redis as _redis_mod

# Central config-service receiver (LIVE-tunable OPERATIONAL knobs). STDLIB-only +
# FAIL-SOFT BY CONTRACT: common.config_client.fetch_config(service, box) returns {}
# on ANY error / timeout / non-200 / bad JSON (NEVER raises), bounded timeout. It is
# OPTIONAL: a pre-cutover image without it must still boot, so the import is guarded
# and the whole live-config layer degrades to today's static env/default behaviour
# when it (or the service) is absent. The package root holds `common/`; pool_worker.py
# sits at that root, so an `import common.config_client` resolves once this file's dir
# is importable (it is — the workers run from the repo root / image WORKDIR).
try:
    from common import config_client as _config_client   # noqa: E402
except Exception:   # noqa: BLE001 — config-service client is OPTIONAL; never break boot
    try:
        import config_client as _config_client           # flat layout fallback (tests add common/ to path)
    except Exception:   # noqa: BLE001
        _config_client = None   # type: ignore[assignment]


def _dbg(consumer: str, task_uuid: str, msg: str) -> None:
    """Step-trace log emitted to BOTH stdout (docker logs) AND syslog
    (journalctl). One channel might be clipped by uvicorn/host buffering;
    the other survives. Cheap — called per major step in the Execute flow.
    Task uuid is short-prefixed for grepability."""
    prefix = f"[pool_worker/{consumer}/{task_uuid[:8]}]"
    line   = f"{prefix} {msg}"
    try:
        print(line, flush=True, file=sys.stdout)
    except Exception:
        pass
    try:
        syslog.syslog(syslog.LOG_INFO, line)
    except Exception:
        pass

# ── State constants ─────────────────────────────────────────────────────────────
#
# 4-state model (session-50). Numbering matches operator's mental model
# FREE → TASK_PICKED → BUSY → FINISHING. Workers write {0, 2, 3} — never 1,
# which is pool-service-only (the lease marker between dispatcher pick and
# the worker's first BUSY heartbeat). The next worker heartbeat overwrites
# TASK_PICKED in Redis with whatever the worker's own state actually is.
# See pool_service.py + SESSION.md memory `project_pool_worker_4_states.md`.

STATE_FREE                       = 0
STATE_TASK_PICKED                = 1   # pool-service only; defined here for symmetry
STATE_BUSY                       = 2
STATE_FINISHING_AND_SHUTTING_DOWN = 3
# FROZEN (session-58 — logical hold). A FROZEN worker is registered + alive +
# heartbeating + READY, but TEMPORARILY refuses new tasks; reversible (thaw). The
# resident model stays loaded. NOT a SIGSTOP (a stopped process can't heartbeat ->
# its memcached ping lapses -> the pool-service reaps it as lost). It rides the
# documented per-worker freeze signal pipeline:freeze:{endpoint} (polled like drain),
# but the terminal action is a HOLD, not a finish-and-vanish. Additive (4); the 0..3
# wire format is unchanged.
STATE_FROZEN                     = 4

# Back-compat name alias for legacy code paths still referring to
# STATE_PREPARINGFORSHUTDOWN (pre-session-50 spelling; the int it resolved
# to was 2, now resolves to 3 — DO NOT compare against the literal 2 in
# legacy code; use the constant).
STATE_PREPARINGFORSHUTDOWN       = STATE_FINISHING_AND_SHUTTING_DOWN

_STATE_NAMES = {
    STATE_FREE:                        "FREE",
    STATE_TASK_PICKED:                 "TASK_PICKED",
    STATE_BUSY:                        "BUSY",
    STATE_FINISHING_AND_SHUTTING_DOWN: "FINISHING_AND_SHUTTING_DOWN",
    STATE_FROZEN:                      "FROZEN",
}

# ── Environment defaults (used when constructor args are None) ──────────────────

POOL_SERVICE_URL   = os.getenv("POOL_SERVICE_URL",      "http://10.121.1.80:8780")
DEFAULT_POOL_ID    = int(os.getenv("WORKER_POOL_ID",    "0"))
# Port UNIQUE PER POOL-WORKER TYPE (operator 2026-07-07): a worker's listen port is derived from its
# pool_id so every worker of a given TYPE binds the SAME deterministic port EVERYWHERE — PORT_BASE +
# pool_id (FP pool2->8802, ocr-image pool9->8809, dissect pool10->8810, assemble pool11->8811, …).
# Retires the all-on-8800 ambiguity (type unreadable from the endpoint) + the qwen sequential-assign
# fragility (pool 8 landing on 8801). Precedence: explicit listen_port arg > WORKER_LISTEN_PORT env
# (full manual override, both kept) > per-type derived default.
_PORT_ENV          = os.getenv("WORKER_LISTEN_PORT")   # None when unset → use the per-type default
PORT_BASE          = int(os.getenv("WORKER_PORT_BASE",  "8800"))
DEFAULT_LISTEN_PORT= int(_PORT_ENV) if _PORT_ENV else PORT_BASE
DEFAULT_CONSUMER   = os.getenv("WORKER_CONSUMER",       "worker-0")
# The heartbeat is a FIXED metronome, set ONCE at launch and NEVER live-tuned or made dynamic:
#   (1) heartbeat < pingback is a SAFETY INVARIANT — a too-slow beat false-evicts a live worker;
#   (2) a dynamically-elastic beat ruins production on load spikes — it feeds back into the load it
#       measures and smears spikes flat (operator, battle-tested). Liveness is not a load signal.
# So PS_HEARTBEAT_INTERVAL / PS_PINGBACK_TIMEOUT live ONLY here as launch constants and are in the
# config-safety set (never reachable from the config-service). The values themselves live in the
# ONE shared source of truth — common.pool_timing — which env-reads the SAME PS_* knobs and ENFORCES
# heartbeat < pingback BY CONSTRUCTION at import (raises on a bad pair), so the pool-service and the
# worker can NEVER drift apart and an unsafe config can never start a worker. A tick is an axiom.
from common.pool_timing import HEARTBEAT_INTERVAL, PINGBACK_TIMEOUT   # noqa: E402
# Heartbeat thread priority (operator 2026-06-23). At normal priority a long CPU-bound
# Execute() (forward_pass / extract on a saturated box) starves the heartbeat thread so its
# beat lapses > PINGBACK_TIMEOUT -> the pool-service false-evicts a LIVE, WORKING worker.
# Run the beat thread at nice -15 so it always gets a slice and last_seen never goes stale.
# Per-thread nice (each pthread is a Linux task) needs CAP_SYS_NICE — the launch scripts add
# it; best-effort otherwise.
HEARTBEAT_NICE = int(os.getenv("PS_HEARTBEAT_NICE", "-15"))
REDIS_URL          = os.getenv("REDIS_URL",             "redis://REDIS_URL-unset.invalid:6379/0")

# ── REDIS_URL is REQUIRED — there is deliberately NO usable default (rm#223) ─────
# Two live Redis instances sit on ADJACENT IPs with identical ports:
#   10.121.1.60  /redis           general (Nextcloud, GitLab, gem, n8n)
#   10.121.1.61  /pipeline-redis  pipeline (pool registry, drain/freeze flags, chains)
# The old default named .60 — which is UP. So a missing REDIS_URL did not fail: it
# connected SUCCESSFULLY to the wrong instance and found an empty pipeline namespace.
# A worker launched without the var registers into .60; pool-service reads .61, never
# sees it, and the worker sits idle forever while reporting itself healthy. Nothing
# errors anywhere — backpressure and total failure look identical.
#
# A wrong default pointing at NOTHING is harmless (crash on connect, fixed in a minute).
# One pointing at a different LIVE store is the rm#218 / derived-stores-diverge-silently
# shape: a plausible wrong answer, not an exception. Hence the sentinel default is an
# RFC-2606 `.invalid` host (can never resolve, and NAMES the cause in the error), and the
# check below refuses to start rather than run against a guess.
#
# Called from the ENTRY POINT, never at import: the test suite imports this module without
# env, and an import-time raise would make the module un-importable (the same reasoning
# that keeps rm#222's sweep under __main__).
REDIS_URL_UNSET_SENTINEL = "REDIS_URL-unset.invalid"


def _require_redis_url(url: str = None) -> str:
    """Fail loudly when REDIS_URL is unset, instead of silently using the wrong live Redis."""
    url = REDIS_URL if url is None else url
    if not url or not url.strip() or REDIS_URL_UNSET_SENTINEL in url:
        raise RuntimeError(
            "REDIS_URL is not set, and it is REQUIRED (rm#223). There is no default on "
            "purpose: the pipeline runs on redis://10.121.1.61:6379/0 (pipeline-redis), "
            "NOT 10.121.1.60 (the general redis) — and .60 is UP, so guessing would "
            "connect successfully to the WRONG instance and silently find an empty "
            "pipeline namespace. Set REDIS_URL (see .env)."
        )
    return url


# ── Central config-service: LIVE-tunable OPERATIONAL knobs (session-59) ──────────
# Every worker can re-read its OPERATIONAL tunables on-flight WITHOUT a redeploy by
# layering the config-service overlay ON TOP of its baked-in env/default. The fetch
# is FAIL-SOFT (returns {} on any error -> env/default wins), BOUNDED, and LAST-GOOD
# cached (a transient miss never flaps a knob back to default). MIRRORS the proven
# stage_watcher overlay discipline. CRITICAL: the fetch is a NETWORK call and runs ONLY
# on a dedicated best-effort refresh thread (NEVER in the heartbeat thread/timer), so a
# hung config-service can never delay a heartbeat and cause a reaper eviction.
CONFIG_FETCH_TIMEOUT     = float(os.getenv("WORKER_CONFIG_TIMEOUT", "4") or "4")   # bound the GET
# Periodic re-fetch cadence, expressed in HEARTBEAT cycles (the worker's natural beat).
# The refresh thread sleeps CONFIG_REFRESH_CYCLES * the live heartbeat interval between
# best-effort fetches; SIGHUP and POST /reload fetch immediately out of band.
CONFIG_REFRESH_CYCLES    = max(1, int(os.getenv("WORKER_CONFIG_REFRESH_CYCLES", "6") or "6"))

# SAFETY/IDENTITY keys that are NEVER live-tunable from the config-service. Identity
# (which pool / box-type / consumer / endpoint) is fixed at launch; the FROZEN/drain
# control rides Redis+env, not the config-service. Stripped from EVERY overlay as a
# belt-and-braces guard so a mistakenly-set row can never reach a worker as one of these.
_CONFIG_SAFETY_KEYS = frozenset({
    "WORKER_POOL_ID", "WORKER_BOX_TYPE", "WORKER_CONSUMER", "WORKER_SERVICE", "WORKER_BOX",
    "WORKER_ENDPOINT", "WORKER_ENDPOINT_IP", "WORKER_LISTEN_PORT",
    "POOL_SERVICE_URL", "REDIS_URL", "GEM_EVENT_URL",
    "CONFIG_SERVICE_URL", "ANDREA_SVC_API_KEY",
    "WORKER_CONFIG_REFRESH_CYCLES", "WORKER_CONFIG_TIMEOUT",
    # Heartbeat/pingback timing is a SAFETY INVARIANT + a metronome (a dynamic beat smears load
    # spikes flat — operator, battle-tested), NEVER an operational knob. Fixed launch constants only.
    "PS_HEARTBEAT_INTERVAL", "PS_PINGBACK_TIMEOUT",
})

# /reload auth: the new authed POST /reload on the worker's existing HTTP server is gated
# by the SAME credential the rest of the stack already mints — the andrea_svc_auth
# X-Agent-Key header, validated constant-time against ANDREA_SVC_API_KEY (the very key
# config_client itself sends to the config-service). The worker's HTTP server carried NO
# prior auth boundary (the freeze/drain controls are Redis-relayed, not inbound), so this
# is the first inbound credential — chosen to match config_client + stage_watcher rather
# than invent a second token. Empty key -> /reload is refused (fail-closed).
CONFIG_RELOAD_KEY_HEADER = "X-Agent-Key"


def _derive_service(consumer: str) -> str:
    """SERVICE name for the config-service. WORKER_SERVICE wins; else derive from the
    consumer by stripping a trailing instance suffix:
        'compressor-tb1-0' -> 'compressor'   'qwen-fp-w2295-3' -> 'qwen-fp'  'tagger' -> 'tagger'
    Rule: drop trailing '-<token>' groups while the token is a pure instance id (an int,
    or a short box/slot tag like w2295/i9v13/tb1/mainbox), keeping the FIRST head segment.
    Conservative: never strips below the first segment, so a bare 'compressor' is unchanged."""
    explicit = os.getenv("WORKER_SERVICE", "").strip()
    if explicit:
        return explicit
    name = (consumer or "").strip() or DEFAULT_CONSUMER
    parts = name.split("-")
    if len(parts) <= 1:
        return name
    _box_tags = {"w2295", "i9v13", "mainbox", "tb1", "tb2", "tb", "mb"}

    def _is_instance_tok(tok: str) -> bool:
        t = tok.strip()
        if not t:
            return True
        if t.isdigit():
            return True                       # pure instance ordinal
        if t.lower() in _box_tags:
            return True                       # short box/slot tag
        # boxN / slotN style ('tb1', 'w0') — a short alpha head + a trailing number
        head = t.rstrip("0123456789")
        return bool(head) and head != t and len(head) <= 3
    while len(parts) > 1 and _is_instance_tok(parts[-1]):
        parts.pop()
    return "-".join(parts) if parts else name


# box subnets -> short box name. service_pool worker IPs: 10.111.1.x = w2295,
# 10.111.2.x = i9v13, 10.121.1.x = mainbox (the source-of-truth thinkbox launch
# scripts assign these). Used only to DERIVE a box label when WORKER_BOX is unset.
_BOX_BY_SUBNET = (
    ("10.111.1.", "w2295"),
    ("10.111.2.", "i9v13"),
    ("10.121.1.", "mainbox"),
)


def _derive_box(endpoint: str) -> str:
    """BOX label for the config-service box overlay. WORKER_BOX wins; else derive a short
    box name from the worker endpoint IP subnet (w2295/i9v13/mainbox). Default '*' (no box
    overlay — the worker gets only the global defaults). Returned to the caller so the
    derivation is surfaced in the init log."""
    explicit = os.getenv("WORKER_BOX", "").strip()
    if explicit:
        return explicit
    ep = (endpoint or "").strip()
    ip = ep.split(":", 1)[0]
    for prefix, name in _BOX_BY_SUBNET:
        if ip.startswith(prefix):
            return name
    return "*"


def _sanitize_overlay(consumer: str, cfg: dict) -> dict:
    """Strip the SAFETY/IDENTITY keys from a fetched overlay (belt-and-braces — they are
    never live-tunable), drop any non-(str/int/float/bool) value (the cfg() caster only
    consumes scalars), and return a clean {k: str}. Mirrors stage_watcher._sanitize_overlay:
    a malformed/forbidden key is DROPPED here so it can never shadow the env/default below it.
    Per-key cast validation is deferred to cfg() so a key usable as a string is still kept."""
    clean = {}
    for k, v in cfg.items():
        if k in _CONFIG_SAFETY_KEYS:
            syslog.syslog(syslog.LOG_INFO,
                          f"[pool_worker/{consumer}] config-service safety key ignored: {k}")
            continue
        if isinstance(v, bool) or isinstance(v, (str, int, float)):
            clean[k] = str(v)
        else:
            syslog.syslog(syslog.LOG_INFO,
                          f"[pool_worker/{consumer}] config-service non-scalar value dropped: {k}")
    return clean

# WORKER_BOX_TYPE — OPTIONAL unsigned-int box-type tag for box-type routing
# (operator 2026-06-15). Set per box on the launch (registry: 0=generic/mainbox,
# 1=i9v13-class, 2=w2295-class). When unset the worker is "untyped" and sends NO
# box_type, so type-constrained tasks skip it. Sent on /register AND every
# heartbeat (both go through _register), satisfying "type at register and heartbeat".
BOX_TYPE = int(os.environ["WORKER_BOX_TYPE"]) if os.environ.get("WORKER_BOX_TYPE", "").strip() else None

# ── DRAIN DECLARATION (rm#486) ────────────────────────────────────────────────────────────────
# Operator 2026-08-23: *"add an auto-discovery for pools, so the advancer will be picking up a new
# pool that identifies itself as 'simple advance me by running my workers'"*.
#
# ⚑ SAME SHAPE AS box_type / mem_total_bytes, deliberately — this file already established the
# pattern and the operator already ratified it (2026-06-22, "advertise both … not uniformly
# filled"): a worker ADVERTISES what it knows, the payload field is OPTIONAL, and the consumer
# gates on whichever is present. Nothing new is invented here; a third optional capability rides
# the same /register call.
#
# ⚑ AND IT RIDES EVERY HEARTBEAT, which is the property that makes it safe. `_heartbeat_loop`
# re-calls `_register`, so the declaration is CONTINUOUSLY RE-ASSERTED rather than latched at
# startup. A pool rolled back to a worker that no longer declares it simply stops being drained,
# with no deregistration step and nothing to clean up. A one-shot declaration would have needed
# one, and the day it was missed the spooler would drive a pool that could no longer honour it.
#
# ⚑ OPT-IN, NOT DEFAULT-ON. Every PoolWorker already signals completion (`_signal_done` is in this
# base class), so it is tempting to declare on everyone's behalf. Do not: what a subclass can still
# get wrong is the STATUS VOCABULARY in its result, and auto-enrolling every pool would hand the
# spooler pools nobody reviewed. Declaring is a statement by the worker's author that its results
# speak `ok`/`retry`/`error` — see rm#486 for what a pool that got this wrong did.
#
#   "simple" = "the spooler may lease one of my tasks and POST /execute; I guarantee a
#               pipeline:stage:done signal carrying that vocabulary."
DRAIN_MODES = ("simple",)
_DRAIN_ENV = os.environ.get("WORKER_DRAIN", "").strip().lower()
DRAIN_MODE = _DRAIN_ENV if _DRAIN_ENV in DRAIN_MODES else None


# WORKER_MEM_TOTAL_BYTES — the box's total RAM, advertised ALONGSIDE box_type so the
# dispatcher can gate routing by a memory-counting formula instead of the (ambiguous)
# box_type NUMBER (operator 2026-06-22: route on real RAM — box_type is a resource-class
# label, NOT the box ordinal, and the qwen/FP workers carried no box_type at all, making
# box_type size-routing inert; a 3M-token doc OOM-killed the 63G box). Read once from
# /proc/meminfo (an unconstrained container sees the host total). Override via the env for
# a memory-LIMITED container. OPTIONAL and rides on /register + every heartbeat: non-uniform
# fill is expected during rollout (operator) — the gate degrades gracefully when absent,
# excluding only boxes KNOWN to be too small.
def _mem_total_bytes():
    env = os.environ.get("WORKER_MEM_TOTAL_BYTES", "").strip()
    if env.isdigit():
        return int(env)
    try:
        with open("/proc/meminfo") as f:
            for line in f:
                if line.startswith("MemTotal:"):
                    return int(line.split()[1]) * 1024   # kB -> bytes
    except Exception:
        pass
    return None


MEM_TOTAL_BYTES = _mem_total_bytes()

# Graceful finish-and-vanish drain (operator 2026-06-16). The pool-service /drain
# endpoint (Bearer-authed) sets a Redis flag `drain:{pool_id}:{endpoint}`; the
# worker POLLS it (no inbound /drain on the worker, no second auth boundary).
# On seeing it the worker stops taking new tasks, finishes the current one, then
# unregisters + exit(0) — with an OPTIONAL last-op gem event so the agent knows to
# relaunch it (typed / new image).
DRAIN_KEY_PREFIX = "pipeline:drain:"   # pipeline:drain:{endpoint} — completes the documented per-worker drain signal
# Logical FROZEN hold (session-58). Mirrors the drain-relay machinery — a durable
# Redis key the worker POLLS (no inbound /freeze on the worker) — but the terminal
# action is a reversible HOLD, not a vanish. While pipeline:freeze:{endpoint} exists
# the worker reports FROZEN (so its ping stays fresh — never reaped) and rejects new
# tasks; deleting the key thaws it back to FREE. Drain WINS over freeze (drain is an
# irreversible exit; freeze is a reversible hold).
FREEZE_KEY_PREFIX = "pipeline:freeze:"
GEM_EVENT_URL    = os.getenv("GEM_EVENT_URL", "http://10.121.1.1:8787/event")
# "ping me back when done" callback POST timeout — bounded so a dead/slow callback endpoint can
# never wedge the worker's post-task path (it runs after _signal_done, before FREE). Best-effort.
CALLBACK_TIMEOUT = float(os.getenv("WORKER_CALLBACK_TIMEOUT", "10") or "10")
try:
    # authenticated afferent emitter (andrea-svc-auth client) — attaches ANDREA_SVC_API_KEY so the
    # gem accepts the event once /event requires auth. Stdlib-only, so the lean worker stays lean.
    from andrea_svc_auth.client import post_event as _svc_post_event
except Exception:   # lib absent in a pre-cutover image — emitters skip (best-effort), never crash the worker
    _svc_post_event = None


# ── IP discovery ────────────────────────────────────────────────────────────────

def _discover_endpoint(listen_port: int) -> str:
    """
    Priority:
      1. WORKER_ENDPOINT  — full override (ip:port)
      2. WORKER_ENDPOINT_IP + listen_port
      3. gethostbyname(gethostname()) + listen_port
    """
    override = os.getenv("WORKER_ENDPOINT", "").strip()
    if override:
        return override

    ip_override = os.getenv("WORKER_ENDPOINT_IP", "").strip()
    if ip_override:
        return f"{ip_override}:{listen_port}"

    ip = socket.gethostbyname(socket.gethostname())
    return f"{ip}:{listen_port}"


# ── Pool service client ─────────────────────────────────────────────────────────

def _ps_call(method: str, path: str, body: dict | None = None) -> bool:
    """Single HTTP call to pool service. Returns True on 2xx. Fails silently."""
    url  = POOL_SERVICE_URL.rstrip("/") + path
    data = json.dumps(body).encode() if body else None
    req  = urllib.request.Request(url, data=data, method=method,
                                  headers={"Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=5) as resp:
            return 200 <= resp.status < 300
    except Exception as exc:
        syslog.syslog(syslog.LOG_DEBUG,
                      f"[pool_worker] PS {method} {path} failed: {exc}")
        return False


# ── HTTP handler ────────────────────────────────────────────────────────────────

class _Handler(http.server.BaseHTTPRequestHandler):
    """Minimal handler. Worker instance injected via server.worker attribute."""

    def log_message(self, fmt, *args):  # silence access log — syslog only
        pass

    def _send(self, code: int, body: dict):
        # BrokenPipeError happens when the client (e.g., docker healthcheck,
        # or a consumer that timed out) closed the connection before we could
        # write the response. That's a NORMAL event for HTTP probes — silently
        # ignore. The 3.14 stdlib's BaseHTTPRequestHandler doesn't swallow it
        # by default, so we wrap the writes.
        data = json.dumps(body).encode()
        try:
            self.send_response(code)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(data)))
            self.end_headers()
            self.wfile.write(data)
        except (BrokenPipeError, ConnectionResetError):
            pass

    def do_GET(self):
        if self.path == "/health":
            worker = self.server.worker
            self._send(200, {
                "status":    "ok",
                "state":     _STATE_NAMES[worker.GetState()],
                "pool_id":   worker._pool_id,
                "endpoint":  worker._endpoint,
                "consumer":  worker._consumer,
            })
        elif self.path == "/ready":
            # ⚑ READINESS, SEPARATE FROM LIVENESS (infra#20). `/health` above answers "is the HTTP
            # server up" and must keep answering 200 whenever it can serve — Docker RESTARTS a
            # container on healthcheck failure, so a liveness probe that fails on a dependency blip
            # bounces every worker at once. That is why this is a NEW route and `/health` is
            # untouched: conflating them turns an observability improvement into an outage
            # amplifier.
            #
            # WHY THIS EXISTS. The pipeline containers' healthcheck is
            # `urlopen('http://localhost:8800/health')` — pure liveness — and this fleet has been
            # bitten by exactly that gap three times: rm#168 ("a stage failing every task at full
            # speed reads as maximally healthy"), rm#288 (pool-service "Up 13 days (healthy)"
            # completing nothing against a 3s median), rm#263 (a thinkbox OFF THE NETWORK while
            # /health returned 200 for ~3h). `stage-stall-watchdog` exists only because this
            # question had no local answer.
            #
            # ⚑ DELIBERATELY CONSERVATIVE. It reports 503 ONLY for states where the worker itself
            # knows it will not take work: FROZEN (admin-frozen) and FINISHING_AND_SHUTTING_DOWN
            # (draining). It does NOT invent a completion-staleness threshold — a worker that has
            # been idle for hours because the QUEUE is empty is perfectly ready, and this file
            # cannot distinguish that from a stall without knowing the stage's median, which lives
            # centrally in stage-stall-watchdog. Asserting more than it can establish is how a
            # readiness probe becomes a restart loop.
            #
            # NOTHING POINTS AT THIS YET. Switching the compose healthchecks over is a separate,
            # deploy-gated change (rm#137 redeploy; rm#298 records the containers currently running
            # stale worker code). Adding the route is additive and cannot affect existing callers.
            worker = self.server.worker
            state = worker.GetState()
            not_taking_work = state in (STATE_FROZEN, STATE_FINISHING_AND_SHUTTING_DOWN)
            self._send(503 if not_taking_work else 200, {
                "ready":     not not_taking_work,
                "state":     _STATE_NAMES[state],
                "reason":    _STATE_NAMES[state] if not_taking_work else "accepting work",
                "pool_id":   worker._pool_id,
                "endpoint":  worker._endpoint,
                "consumer":  worker._consumer,
            })
        elif self.path == "/metrics":
            # Prometheus: per-window forward-pass time + per-job time (operator 2026-06-19).
            try:
                from common.worker_metrics import render
                body, ctype = render()
                self.send_response(200)
                self.send_header("Content-Type", ctype)
                self.send_header("Content-Length", str(len(body)))
                self.end_headers()
                self.wfile.write(body)
            except (BrokenPipeError, ConnectionResetError):
                pass
            except Exception as exc:
                self._send(500, {"error": f"metrics unavailable: {exc}"})
        else:
            self._send(404, {"error": "not found"})

    def do_POST(self):
        """Accept a task: return 200 immediately, do the work in a background
        thread. Per the pool model (operator directive 2026-05-19): the
        consumer dispatches and walks away on 200; it never waits for the
        result. The worker is responsible for storing the result wherever
        the task atom says — `kept` (Redis result key) and/or `awaited`
        (callback_url POST). State transitions worker-side only:
        FREE → BUSY (here, atomically, before spawning the work thread) →
        FREE (in _post_execute_cleanup after Execute()).

        Also serves POST /reload: an AUTHED (X-Agent-Key == ANDREA_SVC_API_KEY,
        constant-time, fail-closed) trigger to re-fetch the live config-service overlay
        out of band (same effect as SIGHUP). It NEVER runs the network fetch on the HTTP
        thread synchronously beyond config_client's own bounded timeout, and it cannot
        raise into the server (the refresh is fully wrapped).
        """
        if self.path == "/reload":
            self._handle_reload()
            return
        if self.path != "/execute":
            self._send(404, {"error": "not found"})
            return

        length = int(self.headers.get("Content-Length", 0))
        body   = json.loads(self.rfile.read(length)) if length else {}
        task_uuid = body.get("task_uuid", "").strip()

        if not task_uuid:
            self._send(400, {"error": "task_uuid required"})
            return

        worker = self.server.worker

        # Atomically claim: only one POST can flip FREE → BUSY. Concurrent
        # POSTs see BUSY and get 503.
        with worker._state_lock:
            if worker._draining:
                self._send(503, {"error": "worker draining"})
                return
            if worker._frozen:
                # Logical FROZEN hold (session-58): registered + alive + READY but
                # temporarily NOT accepting tasks. Reject the pickup; the worker stays
                # resident and instantly re-dispatchable the moment it thaws. The
                # pool-service already excludes us via pipeline:freeze:{endpoint}, so this
                # is the belt to the get_worker suspenders (covers an in-flight dispatch
                # that raced the freeze, or a direct /execute).
                self._send(503, {"error": "worker frozen"})
                return
            if worker._state != STATE_FREE:
                self._send(503, {"error": "worker busy"})
                return
            worker._state = STATE_BUSY
        # Notify pool-service of the transition (outside the lock — the
        # network call shouldn't hold the state lock).
        worker._register(STATE_BUSY)

        # Spawn the actual work in a background thread. The HTTP handler
        # returns immediately; the connection closes. The consumer is now
        # free to move on to its next dispatch — it never sees the result
        # through this connection.
        threading.Thread(
            target=worker._execute_async,
            args=(task_uuid,),
            daemon=True,
            name=f"execute-{task_uuid[:8]}",
        ).start()

        self._send(200, {"accepted": True, "task_uuid": task_uuid})

    def _handle_reload(self):
        """POST /reload — AUTHED out-of-band live-config re-fetch (same effect as SIGHUP).
        Auth: X-Agent-Key must equal ANDREA_SVC_API_KEY, compared in CONSTANT TIME, FAIL-
        CLOSED (an empty server key, or a missing/mismatched header -> 403). This is the
        worker HTTP server's ONLY inbound credential; it matches what config_client already
        mints, so consumers need no new key. The refresh itself is fully wrapped (it can
        never raise) and bounded, so this handler cannot wedge the server."""
        worker = self.server.worker
        import hmac
        server_key = os.getenv("ANDREA_SVC_API_KEY", "")
        presented  = self.headers.get(CONFIG_RELOAD_KEY_HEADER, "") or ""
        # fail-closed: no server key configured -> /reload is disabled, not open.
        if not server_key or not hmac.compare_digest(str(presented), str(server_key)):
            self._send(403, {"error": "forbidden"})
            return
        changed = worker._refresh_config(reason="reload")
        self._send(200, {"reloaded": True, "changed": bool(changed),
                         "service": worker._cfg_service, "box": worker._cfg_box})


# ── Pool worker base class ──────────────────────────────────────────────────────

class PoolWorker(ABC):
    """
    Abstract base class for pool-registered workers.

    Subclasses must implement:
        Execute(task_uuid: str) -> bool
        on_cleanup()  [optional, default: no-op]
    """

    def __init__(self,
                 pool_id: int | None = None,
                 listen_port: int | None = None,
                 consumer: str | None = None,
                 install_signal_handlers: bool = True,
                 default_callback: dict | None = None,
                 drain: str | None = None):
        gc.set_threshold(700, 10, 5)

        # ── "ping me back when you're done" (operator 2026-07-11) — the whole event bus,
        # as simple as it can be: no broker, just a callback descriptor threaded through the
        # request. A caller that dispatches work can attach, in the task atom's `callback`
        # (or the legacy `callback_url` shorthand), WHERE to POST the completion event + a
        # verbatim CONTEXT it wants echoed back:
        #     "callback": {"url": "http://…/event", "event": "FileParsedToText",
        #                  "context": {"session_id": 1}}      // context may be any JSON (list/dict)
        # On completion this base class POSTs that event to that endpoint for EVERY descendant,
        # so e.g. a crawl that hands a fetched PDF to the handler says "ping me back when done,
        # session id is 1, the event is FileParsedToText" and gets exactly that event fired at
        # its endpoint. `default_callback` here is a worker-LEVEL fallback (a descriptor merged
        # UNDER each task's own callback) for a worker that always reports to a fixed endpoint;
        # the per-task atom callback wins key-by-key. Empty {} = no default (pure per-task).
        self._default_callback = dict(default_callback) if default_callback else {}

        # rm#486 — the drain declaration. Explicit arg wins over WORKER_DRAIN so a subclass can
        # state it in code (the honest place: the author of the Execute() that produces the status
        # vocabulary is the one who can promise it), while the env stays available for a rollout
        # that wants to arm/disarm without a code change. An unrecognised value is IGNORED rather
        # than passed through — an unknown drain mode reaching the pool service would be a
        # capability claim nobody can honour, and silently declaring one is the failure this
        # mechanism exists to prevent.
        _d = (drain or "").strip().lower() or None
        self._drain = _d if _d in DRAIN_MODES else (None if _d else DRAIN_MODE)
        if _d and _d not in DRAIN_MODES:
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker] IGNORING unknown drain mode {drain!r} — known: {DRAIN_MODES}. "
                          f"This worker will NOT be auto-discovered for spooler dispatch.")

        self._pool_id     = pool_id     if pool_id     is not None else DEFAULT_POOL_ID
        # Port unique per pool-worker TYPE: explicit arg > WORKER_LISTEN_PORT env > PORT_BASE+pool_id.
        if listen_port is not None:
            self._listen_port = listen_port
        elif _PORT_ENV is not None:
            self._listen_port = DEFAULT_LISTEN_PORT
        else:
            self._listen_port = PORT_BASE + self._pool_id
        self._consumer    = consumer    if consumer    is not None else DEFAULT_CONSUMER

        self._state_lock = threading.Lock()
        self._state      = STATE_FREE
        self._running    = True
        self._draining   = False        # set once a PS-relayed drain flag is seen; commits to finish-and-vanish
        self._frozen     = False        # set while a PS-relayed freeze flag is seen; reversible HOLD (model stays resident)
        self._drain_at_checkpoint = False   # upgrade protocol: in-flight compute stops at its next checkpoint
        self._upgrade_lock_doc    = None    # upgrade protocol: doc to lock to this endpoint on the upgrade-unregister
        self._endpoint   = _discover_endpoint(self._listen_port)

        # ── LIVE config-service layer (session-59) ──────────────────────────────────
        # SERVICE = WORKER_SERVICE else the consumer with its trailing instance suffix
        # stripped; BOX = WORKER_BOX else derived from the endpoint subnet (else '*').
        # _cfg_overlay holds the LAST-GOOD, sanitized {k: str} from the config-service;
        # cfg(key, default, cast) layers overlay > env > default. The fetch is wrapped so
        # it can NEVER raise into a loop, bounded, and last-good cached. CRITICAL: it runs
        # ONLY on the dedicated refresh thread / SIGHUP / POST /reload — NEVER on the
        # heartbeat thread (see _heartbeat_loop + _config_refresh_loop).
        self._cfg_service = _derive_service(self._consumer)
        self._cfg_box     = _derive_box(self._endpoint)
        self._cfg_lock    = threading.Lock()
        self._cfg_overlay: dict = {}        # last-good sanitized overlay (empty until first good fetch)

        # Redis for task atom access. socket_timeout=10 (was 60 — audit 2026-06-19): the
        # heartbeat loop's drain-check rides this client, and a 60s stall there blew past
        # PINGBACK_TIMEOUT(30s) and got a LIVE worker evicted. Atom reads/writes are small,
        # so 10s is ample; the liveness refresh is also reordered ahead of the drain-read.
        self._redis = _redis_mod.from_url(
            _require_redis_url(),
            decode_responses=True,
            socket_connect_timeout=5,
            socket_timeout=10,
            retry_on_timeout=True,
        )
        self._redis.ping()

        # syslog.openlog is process-global — multiple PoolWorker instances will
        # rewrite the ident. Last one wins. Use the consumer prefix in every
        # log line via the f-strings below to keep per-worker attribution clear.
        syslog.openlog(f"pool_worker/{self._consumer}", syslog.LOG_PID, syslog.LOG_DAEMON)
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] init "
                      f"endpoint={self._endpoint} pool={self._pool_id} "
                      f"cfg_service={self._cfg_service} cfg_box={self._cfg_box} "
                      f"config_client={'present' if _config_client is not None else 'absent'}")

        # STARTUP config fetch — best-effort + bounded + fully wrapped (never raises). On
        # the constructor thread (NOT the heartbeat thread, which hasn't started yet), so a
        # slow config-service at boot delays only init, never a live beat. A miss leaves the
        # overlay empty -> cfg() falls through to env/default exactly as before.
        self._refresh_config(reason="startup")

        # Register with pool service
        self._register(STATE_FREE)

        # Heartbeat thread
        self._hb_thread = threading.Thread(target=self._guarded(self._heartbeat_loop, "heartbeat"),
                                           daemon=True,
                                           name=f"heartbeat-{self._consumer}")
        self._hb_thread.start()

        # Config refresh thread — DEDICATED, SEPARATE from the heartbeat thread (HEARTBEAT
        # SAFETY, requirement 3): the config fetch is a network call, so it must NEVER share
        # the heartbeat thread; a hung config-service can then delay only THIS thread, never a
        # beat -> the worker is never falsely reaped. It re-fetches every
        # CONFIG_REFRESH_CYCLES * (live heartbeat interval) seconds, best-effort.
        self._cfg_thread = threading.Thread(target=self._guarded(self._config_refresh_loop, "config-refresh", fatal=False),
                                            daemon=True,
                                            name=f"config-refresh-{self._consumer}")
        self._cfg_thread.start()

        # HTTP server. LISTEN host — a worker LISTENS EXACTLY WHERE IT ADVERTISES. A host-native worker
        # must NEVER bind 0.0.0.0: the mainbox host INPUT chain is policy-accept, so 0.0.0.0 = the public
        # NIC (Andrea#117 / rm#134). Precedence: explicit WORKER_LISTEN_HOST > the advertised
        # WORKER_ENDPOINT_IP (you declared your own reachable IP -> bind it) > "0.0.0.0". The 0.0.0.0
        # fallback is correct+safe ONLY for a container on svc_pool (0.0.0.0 is namespaced to 10.121.1.0/24)
        # and for boxes that advertise via gethostbyname/WORKER_ENDPOINT — those keep the prior behavior.
        # A containerized worker that advertises a HOST ip while binding in-container must set
        # WORKER_LISTEN_HOST=0.0.0.0 explicitly. rules/host-native-services-bind-svcpool-ip
        self._listen_host = (os.getenv("WORKER_LISTEN_HOST", "").strip()
                             or os.getenv("WORKER_ENDPOINT_IP", "").strip()
                             or "0.0.0.0")
        self._server = http.server.HTTPServer((self._listen_host, self._listen_port), _Handler)
        self._server.worker = self

        # Signal handlers (process-scoped — last installer wins).
        # When multiple PoolWorker instances live in one process, the host
        # must orchestrate drain itself and pass install_signal_handlers=False.
        if install_signal_handlers:
            signal.signal(signal.SIGTERM, self._on_drain)
            signal.signal(signal.SIGINT,  self._on_drain)
            signal.signal(signal.SIGHUP,  self._on_reload)
            signal.signal(signal.SIGUSR1, self._on_gc)
            signal.signal(signal.SIGUSR2, self._on_dump)

    # ── Public API ─────────────────────────────────────────────────────────────

    def GetState(self) -> int:
        with self._state_lock:
            return self._state

    # ── LIVE config layer ───────────────────────────────────────────────────────
    def cfg(self, key: str, default, cast=str):
        """Return the LIVE value for an OPERATIONAL tunable, precedence:
            config-service overlay  >  env  >  the passed default
        `cast` is one of str / int / float / bool (the type to coerce to). A value that
        fails the cast is DROPPED (falls through to the next source) — exactly the
        stage_watcher 'drop-malformed -> keep prior/default' discipline, so a poisoned
        live value can never crash a caller. NEVER raises. SAFETY/IDENTITY keys are not
        here at all (they were stripped from the overlay); pass them via env only.

        bool casting: 1/true/yes/on -> True, 0/false/no/off -> False (case-insensitive);
        anything else is malformed -> drop. The passed `default` is returned VERBATIM
        (already the caller's type), never re-cast."""
        with self._cfg_lock:
            ov = self._cfg_overlay.get(key)
        if ov is not None:
            coerced = self._coerce(ov, cast)
            if coerced is not None:
                return coerced
            # malformed live value -> fall through to env/default (drop)
        env = os.getenv(key)
        if env is not None:
            coerced = self._coerce(env, cast)
            if coerced is not None:
                return coerced
        return default

    @staticmethod
    def _coerce(raw, cast):
        """Coerce a str to cast; return None on failure (caller falls through). Never raises."""
        try:
            s = str(raw).strip()
            if cast is bool:
                low = s.lower()
                if low in ("1", "true", "yes", "on"):
                    return True
                if low in ("0", "false", "no", "off"):
                    return False
                return None
            if cast is int:
                return int(s)
            if cast is float:
                return float(s)
            return s                      # str (or any other passthrough)
        except (TypeError, ValueError):
            return None

    def _refresh_config(self, reason: str = "") -> bool:
        """Fetch the config-service overlay for this service+box and swap it in as the new
        last-good. FAIL-SOFT + BOUNDED + LAST-GOOD-CACHED + fully WRAPPED so it can NEVER
        raise into any loop/thread/handler. Returns True iff the live overlay CHANGED.
        MIRRORS stage_watcher._service_overlay:
          * client missing / fetch raises / returns {} (service down/slow) -> KEEP last-good
            (a transient miss never flaps a knob back to default)
          * a non-empty fetch -> sanitize (strip safety keys + non-scalars), then:
              - nothing usable left -> KEEP last-good
              - usable + DIFFERENT  -> swap in + log 'applied' (only on CHANGE)
        This is the ONLY place that performs the network fetch; the heartbeat thread never
        calls it. config_client.fetch_config is itself fail-soft + bounded, so this method
        cannot block beyond CONFIG_FETCH_TIMEOUT."""
        try:
            if _config_client is None:
                return False
            try:
                box = None if self._cfg_box == "*" else self._cfg_box
                raw = _config_client.fetch_config(self._cfg_service, box=box,
                                                  timeout=CONFIG_FETCH_TIMEOUT)
            except Exception as exc:   # noqa: BLE001 — fetch_config is fail-soft; pure belt-and-braces
                syslog.syslog(syslog.LOG_DEBUG,
                              f"[pool_worker/{self._consumer}] config fetch error ({reason}): {exc}")
                return False
            if not isinstance(raw, dict) or not raw:
                return False           # {} = down/slow/empty -> hold last-good (no flap)
            clean = _sanitize_overlay(self._consumer, raw)
            if not clean:
                return False           # nothing usable left -> hold last-good (no flap)
            with self._cfg_lock:
                changed = clean != self._cfg_overlay
                if changed:
                    self._cfg_overlay = clean
            if changed:
                applied = ",".join(f"{k}={v}" for k, v in sorted(clean.items()))
                syslog.syslog(syslog.LOG_INFO,
                              f"[pool_worker/{self._consumer}] config-service applied "
                              f"({reason}) service={self._cfg_service} box={self._cfg_box}: {applied}")
            return changed
        except Exception as exc:       # noqa: BLE001 — ABSOLUTE guard: must never raise into a caller
            try:
                syslog.syslog(syslog.LOG_WARNING,
                              f"[pool_worker/{self._consumer}] _refresh_config guard caught: {exc}")
            except Exception:
                pass
            return False

    def _config_refresh_loop(self):
        """Dedicated daemon thread (NOT the heartbeat thread) that periodically re-fetches
        the live config overlay. Sleeps CONFIG_REFRESH_CYCLES * the FIXED heartbeat interval
        between best-effort fetches. Every fetch is wrapped (never raises); a hang in the fetch
        blocks ONLY this thread, never a heartbeat. Exits when _running clears."""
        while self._running:
            beat = HEARTBEAT_INTERVAL   # FIXED — the heartbeat is never live/dynamic
            nap = max(1, int(beat)) * CONFIG_REFRESH_CYCLES
            # sleep in 1s slices so a drain/shutdown is observed promptly (and so the thread
            # never holds the process open past _running going false).
            slept = 0
            while slept < nap and self._running:
                time.sleep(1)
                slept += 1
            if not self._running:
                break
            self._refresh_config(reason="periodic")

    @abstractmethod
    def Execute(self, task_uuid: str) -> bool:
        """
        Perform the task identified by task_uuid.
        BUSY is already set before this is called.
        Return True on success, False on failure.
        Do NOT set state here — the framework handles it.
        """
        raise NotImplementedError

    def on_cleanup(self):
        """
        Release task-local resources after Execute completes.
        Called before GC and before transitioning to FREE.
        Default: no-op. Override to release tensors, file handles, etc.
        Do NOT unload persistent resources (models, DB connections) here.
        """

    def on_drain(self):
        """Called when drain signal received. Override for custom drain logic."""

    def serve(self):
        """Start serving HTTP requests. Blocks until stopped."""
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] serving on "
                      f"{self._listen_host}:{self._listen_port}")
        try:
            self._server.serve_forever()
        except Exception:
            pass
        self._shutdown()

    def shutdown(self):
        """Public drain entrypoint for embedded use (host orchestrates signals)."""
        self._on_drain(0, None)

    # ── Internal ───────────────────────────────────────────────────────────────

    def _set_state(self, state: int):
        with self._state_lock:
            self._state = state
        # Immediately fire heartbeat with new state — don't wait for interval
        self._register(state)

    def _execute_async(self, task_uuid: str) -> None:
        """
        Run Execute() in a background thread. State is already BUSY when we
        enter (set atomically by do_POST). On completion, POST the result
        to the task atom's callback_url if one was specified ("awaited"
        contract), then run cleanup which transitions BUSY → FREE.

        Every step is _dbg()-traced so a silent disappearance (e.g.,
        Pride and Prejudice 2026-05-20) leaves a forensic trail.
        """
        _dbg(self._consumer, task_uuid, "ENTER _execute_async")
        ok = False
        _job_t0 = time.monotonic()
        try:
            _dbg(self._consumer, task_uuid, "calling Execute()")
            ok = self.Execute(task_uuid)
            _dbg(self._consumer, task_uuid, f"Execute() returned ok={ok}")
        except Exception as exc:
            tb = traceback.format_exc()
            syslog.syslog(syslog.LOG_ERR,
                          f"[pool_worker/{self._consumer}] Execute({task_uuid}) raised: {exc}")
            _dbg(self._consumer, task_uuid,
                 f"Execute() RAISED: {type(exc).__name__}: {exc}\n{tb}")
            ok = False
            # Write a synthetic error result so callers polling task:{uuid}:result
            # see SOMETHING rather than waiting forever for a result that's not coming.
            try:
                self._redis.set(
                    f"task:{task_uuid}:result",
                    json.dumps({"status": "error", "error": str(exc),
                                "out_stage": "", "out_key": ""}),
                    ex=86400,
                )
                _dbg(self._consumer, task_uuid, "synthetic error result written")
            except Exception as exc2:
                _dbg(self._consumer, task_uuid,
                     f"synthetic error result write FAILED: {type(exc2).__name__}: {exc2}")

        # per-job time (operator 2026-06-19): "time per job" for every stage worker,
        # labelled by pool/stage. Exposed via the worker's /metrics endpoint.
        try:
            from common.worker_metrics import WORKER_JOB_SECONDS
            WORKER_JOB_SECONDS.labels(stage=str(self._pool_id)).observe(time.monotonic() - _job_t0)
        except Exception:
            pass

        # Signal completion by writing to a Redis stream. The consumer of
        # that stream (e.g., the dispatcher's stage-advance task) picks it up
        # on its own poll loop — no synchronous HTTP from the worker side.
        # This is the "kept with auto-pickup" pattern: write to a known
        # place, whoever's watching it picks the work up. Worker never holds
        # a network call to an external caller's lifetime.
        try:
            _dbg(self._consumer, task_uuid, "calling _signal_done()")
            self._signal_done(task_uuid)
            _dbg(self._consumer, task_uuid, "_signal_done() returned cleanly")
        except Exception as exc:
            tb = traceback.format_exc()
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker/{self._consumer}] signal_done failed for {task_uuid}: {exc}")
            _dbg(self._consumer, task_uuid,
                 f"_signal_done() RAISED: {type(exc).__name__}: {exc}\n{tb}")

        # "awaited" contract — the operator's event bus: if the caller attached a callback
        # descriptor to the task ("ping me back when done, session_id=1, event=FileParsedToText"),
        # POST that completion event to its endpoint now. Fully wrapped + bounded — a dead
        # callback endpoint must never wedge the post-task path or block the BUSY→FREE transition.
        try:
            self._fire_callback(task_uuid, ok)
        except Exception as exc:
            tb = traceback.format_exc()
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker/{self._consumer}] fire_callback failed for {task_uuid}: {exc}")
            _dbg(self._consumer, task_uuid,
                 f"_fire_callback() RAISED: {type(exc).__name__}: {exc}\n{tb}")

        _dbg(self._consumer, task_uuid, "calling _post_execute_cleanup()")
        try:
            self._post_execute_cleanup()
        except Exception as exc:
            tb = traceback.format_exc()
            _dbg(self._consumer, task_uuid,
                 f"_post_execute_cleanup() RAISED: {type(exc).__name__}: {exc}\n{tb}")
        _dbg(self._consumer, task_uuid, "EXIT _execute_async")

    def _signal_done(self, task_uuid: str) -> None:
        """Write a completion-signal entry to the Redis stream specified in
        the task atom (atom['signal_stream']; defaults to pipeline:stage:done
        for the Andrea pipeline). The entry carries task_uuid + a copy of the
        result-key fields so the consumer doesn't need to fetch separately
        in the common case (but it still can, for the source of truth)."""
        _dbg(self._consumer, task_uuid, "_signal_done: GET task atom")
        raw = self._redis.get(f"task:{task_uuid}")
        if not raw:
            _dbg(self._consumer, task_uuid,
                 "_signal_done: NO task atom in redis — skipping XADD (atom may have expired)")
            return
        atom = json.loads(raw)
        signal_stream = (atom.get("signal_stream") or "pipeline:stage:done").strip()
        _dbg(self._consumer, task_uuid,
             f"_signal_done: GET task:result  signal_stream='{signal_stream}'")
        result_raw = self._redis.get(f"task:{task_uuid}:result")
        if result_raw:
            result = json.loads(result_raw)
            _dbg(self._consumer, task_uuid,
                 f"_signal_done: result-key present  status={result.get('status')} "
                 f"out_stage={result.get('out_stage')} out_key={result.get('out_key','')[:60]}")
        else:
            result = {"status": "error", "error": "no result written"}
            _dbg(self._consumer, task_uuid,
                 "_signal_done: result-key MISSING — synthesizing status=error")
        fields = {
            "task_uuid": task_uuid,
            "status":    str(result.get("status", "error")),
            "out_stage": str(result.get("out_stage", "")),
            "out_key":   str(result.get("out_key", "")),
            "error":     str(result.get("error", "")),
        }
        # token_count rides the done signal when the stage reported it (tokenize),
        # so the drainer persists it and the advancer can size-route — no Garage read.
        _tc = result.get("token_count")
        if _tc is not None:
            fields["token_count"] = str(_tc)
        # Cap stream length so it doesn't grow unbounded; approximate=True for
        # speed (XADD MAXLEN ~ uses radix-tree-friendly trimming).
        _dbg(self._consumer, task_uuid,
             f"_signal_done: XADD → '{signal_stream}' (status={fields['status']})")
        msg_id = self._redis.xadd(signal_stream, fields, maxlen=10000, approximate=True)
        _dbg(self._consumer, task_uuid,
             f"_signal_done: XADD returned msg_id={msg_id}")

    def _resolve_callback(self, atom: dict) -> dict | None:
        """Merge the worker-level default callback with THIS task's callback descriptor and
        normalize it → {url, event, context} (or None if there's nowhere to ping).

        Accepted in the atom, most-specific wins key-by-key over `self._default_callback`:
          * atom['callback']      = {"url","event","context"}   — the full descriptor
          * atom['callback_url']  = "http://…"                   — legacy string shorthand (url only)
        `context` is echoed back VERBATIM (any JSON — the operator's 'optional list attached to
        the event', e.g. {"session_id": 1}). `event` names the event ('FileParsedToText')."""
        cb = dict(self._default_callback)
        raw = atom.get("callback")
        if isinstance(raw, dict):
            cb.update({k: v for k, v in raw.items() if v is not None})
        url_shorthand = atom.get("callback_url")
        if isinstance(url_shorthand, str) and url_shorthand.strip():
            cb["url"] = url_shorthand.strip()
        url = str(cb.get("url", "")).strip()
        if not url:
            return None
        cb["url"] = url
        return cb

    def _fire_callback(self, task_uuid: str, ok: bool) -> None:
        """POST the completion EVENT to the caller-specified endpoint (the 'awaited' half of the
        pool contract — the event bus). No-op when the task carried no callback. Best-effort +
        BOUNDED (CALLBACK_TIMEOUT) so a dead endpoint can't wedge the worker; authed with the same
        X-Agent-Key the rest of the stack mints, when ANDREA_SVC_API_KEY is set.

        The POSTed body:
            {"event": <name|"TaskDone">, "task_uuid", "ok": bool, "status", "out_stage",
             "out_key", "error", "pool_id", "endpoint", "context": <echoed verbatim>}"""
        raw = self._redis.get(f"task:{task_uuid}")
        if not raw:
            _dbg(self._consumer, task_uuid, "_fire_callback: no task atom — skip")
            return
        atom = json.loads(raw)
        cb = self._resolve_callback(atom)
        if not cb:
            return                                  # nothing to ping — the common (no-callback) path
        result = {}
        result_raw = self._redis.get(f"task:{task_uuid}:result")
        if result_raw:
            try:
                result = json.loads(result_raw)
            except Exception:                       # noqa: BLE001
                result = {}
        body = {
            "event":     str(cb.get("event") or "TaskDone"),
            "task_uuid": task_uuid,
            "ok":        bool(ok),
            "status":    str(result.get("status", "ok" if ok else "error")),
            "out_stage": str(result.get("out_stage", "")),
            "out_key":   str(result.get("out_key", "")),
            "error":     str(result.get("error", "")),
            "pool_id":   self._pool_id,
            "endpoint":  self._endpoint,
            "context":   cb.get("context"),         # echoed verbatim — the caller's session id etc.
        }
        headers = {"Content-Type": "application/json"}
        svc_key = os.getenv("ANDREA_SVC_API_KEY", "")
        if svc_key:
            headers["X-Agent-Key"] = svc_key
        _dbg(self._consumer, task_uuid,
             f"_fire_callback: POST event='{body['event']}' -> {cb['url']} "
             f"context={json.dumps(cb.get('context'))[:80]}")
        req = urllib.request.Request(cb["url"], data=json.dumps(body).encode(),
                                     headers=headers, method="POST")
        try:
            with urllib.request.urlopen(req, timeout=CALLBACK_TIMEOUT) as resp:
                _dbg(self._consumer, task_uuid,
                     f"_fire_callback: endpoint returned {resp.status}")
        except urllib.error.HTTPError as exc:
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker/{self._consumer}] callback {cb['url']} HTTP {exc.code}")
        except Exception as exc:                    # noqa: BLE001 — best-effort; never break the worker
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker/{self._consumer}] callback {cb['url']} failed: {exc}")

    def _post_execute_cleanup(self):
        """on_cleanup → GC → FREE. Called after every Execute()."""
        try:
            self.on_cleanup()
        except Exception as exc:
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker/{self._consumer}] on_cleanup raised: {exc}")
        gc.disable()
        gc.collect(2)
        gc.enable()
        # If a drain was requested while this task ran, vanish NOW — BEFORE going
        # back to FREE, so get_worker can't grab us in the gap (finish-and-vanish).
        # Drain WINS over freeze: an irreversible exit takes precedence over a hold.
        if self._draining or self._check_drain() is not None:
            self._draining = True
            self._vanish()                  # os._exit(0) — never returns
        # If a freeze was requested while this task ran (or is still in effect), the
        # freeze "takes effect when the task finishes" — settle into FROZEN, not FREE,
        # so the next get_worker can't grab us between task-done and the next beat. The
        # model stays resident; thaw flips us back to FREE.
        if self._frozen or self._check_freeze():
            self._frozen = True
            self._set_state(STATE_FROZEN)
            return
        self._set_state(STATE_FREE)

    def _register(self, state: int) -> bool:
        payload = {
            "endpoint": self._endpoint,
            "state":    state,
            "pool_id":  self._pool_id,
        }
        # box_type AND mem_total_bytes both ride on /register and every heartbeat (the
        # heartbeat loop re-calls _register). Both OPTIONAL — a worker advertises whatever
        # it knows; the dispatcher gates on whichever is present (operator 2026-06-22:
        # "advertise both: resource-class and the amount of memory; not uniformly filled").
        if BOX_TYPE is not None:
            payload["box_type"] = BOX_TYPE
        if MEM_TOTAL_BYTES is not None:
            payload["mem_total_bytes"] = MEM_TOTAL_BYTES
        # rm#486: the drain declaration, same OPTIONAL discipline — absent means "do not
        # auto-discover me", which is the safe default for every pool that predates this.
        if self._drain:
            payload["drain"] = self._drain
        return _ps_call("POST", "/register", payload)

    def _unregister(self) -> bool:
        body = {"endpoint": self._endpoint, "pool_id": self._pool_id}
        # Upgrade protocol (operator 2026-06-19): if we drained a checkpointable task for an
        # upgrade, tell PS to LOCK that doc to our endpoint, so the relaunched worker (same IP,
        # new image) resumes it from its own Garage checkpoint — no cross-worker race, no hang.
        if self._upgrade_lock_doc:
            body["upgrade"]  = True
            body["lock_doc"] = self._upgrade_lock_doc
        return _ps_call("POST", "/unregister", body)

    # ── Graceful drain (PS-relay via Redis flag) ─────────────────────────────────
    def _drain_key(self) -> str:
        return f"{DRAIN_KEY_PREFIX}{self._endpoint}"

    def _check_drain(self):
        """Return the drain directive {notify,reason} if THIS worker's drain flag
        (pipeline:drain:{endpoint}) is set, else None. Polled — no inbound to the
        worker, no second auth boundary. Per-worker only: the system pipeline:drain
        monitoring flag is intentionally NOT a mass-vanish trigger."""
        try:
            raw = self._redis.get(self._drain_key())
            if not raw:
                return None
            try: return json.loads(raw)
            except Exception: return {"notify": False, "reason": "drain"}
        except Exception:
            return None

    # ── Logical FROZEN hold (PS-relay via Redis flag) ────────────────────────────
    def _freeze_key(self) -> str:
        return f"{FREEZE_KEY_PREFIX}{self._endpoint}"

    def _check_freeze(self) -> bool:
        """True if THIS worker's freeze flag (pipeline:freeze:{endpoint}) is present.
        Polled in the heartbeat loop, exactly like the drain-check — no inbound to the
        worker. A read error returns False (best-effort): the freeze is a soft hold, so
        a transient Redis blip must never wedge the worker; the next beat re-reads."""
        try:
            return bool(self._redis.exists(self._freeze_key()))
        except Exception:
            return False

    def _vanish(self):
        """Finish-and-vanish: the current task is done (state FREE). Unregister,
        clear the flag, fire the OPTIONAL last-op gem event, then exit(0)."""
        info = self._check_drain() or {}
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] drain complete — vanishing "
                      f"(notify={bool(info.get('notify'))})")
        self._running = False
        try: self._unregister()
        except Exception: pass
        try: self._redis.delete(self._drain_key())   # don't strand a stale flag for the next worker at this endpoint
        except Exception: pass
        if info.get("notify"):
            self._post_gem_vanish_event(info.get("reason", ""))
        os._exit(0)

    # Distinct, greppable marker. The exit CODE tells a supervisor what to do; this line tells
    # a human (and log-watch) WHAT HAPPENED. Keep the token stable — alerting keys on it.
    CRASH_MARKER = "POOL_WORKER_CRASH"

    def _die(self, reason: str, tb: str = "", code: int = 70):
        """Abnormal exit: NON-ZERO, loudly, with a signature the logs can be searched for.

        WHY THIS EXISTS (operator, 2026-07-27: "код выхода почини — чтобы авария от слива
        отличалась, но это одним лишь кодом не решается — анализ логов is a must").

        Before this, EVERY way out of a worker was exit 0: `_vanish()` for a drain by design,
        and an unhandled error too, because the exception unwound into uvicorn which shut down
        gracefully. Docker's `restart=on-failure` — the fleet default on nearly every pool
        worker — fires only on a NON-ZERO exit, so a crash and a deliberate drain were literally
        the same event to the supervisor and neither came back.

        Measured: qwen-pool ended a transient `redis.TimeoutError` with rc=0 and stayed down 25
        hours with 3457 documents dispatchable, on a box at load 0.1.

        Code 70 = EX_SOFTWARE (sysexits.h) — an internal fault, not a bad invocation (64) and
        not an unavailable dependency (69); a supervisor or a human reading `docker inspect`
        gets a category, not just "non-zero". We do NOT unregister here: a crashed worker's
        registration expiring on its own ping timeout is the truthful signal to pool-service,
        whereas a tidy /unregister would claim an orderly departure that did not happen.
        """
        line = (f"[pool_worker/{self._consumer}] {self.CRASH_MARKER} exit={code} "
                f"reason={reason} endpoint={self._listen_host}:{self._listen_port} "
                f"pool={self._pool_id}")
        syslog.syslog(syslog.LOG_ERR, line)
        # ALSO to stderr, and that is not belt-and-braces. syslog() goes to /dev/log, whose
        # stream log-watch's selector deliberately EXCLUDES (service_name!~".*syslog.*", added
        # to stop a ganglia flood, drop-in 2026-07-18). stderr lands in the container's own
        # json-file log, which fluent-bit ships under the CONTAINER's service_name — the stream
        # the selector actually watches, and the one `docker logs` shows a human. A crash marker
        # only in the excluded stream would be invisible to exactly the tooling meant to catch it.
        try:
            print(line, file=sys.stderr, flush=True)
        except Exception:                       # noqa: BLE001 — a closed stderr must not block the exit
            pass
        if tb:
            for line in tb.strip().splitlines()[-12:]:
                syslog.syslog(syslog.LOG_ERR, f"[pool_worker/{self._consumer}] {line}")
        try:
            self._post_gem_crash_event(reason, tb)
        except Exception:                       # noqa: BLE001 — never let alerting block the exit
            pass
        os._exit(code)

    def _post_gem_vanish_event(self, reason: str) -> None:
        """Last op before exit(0): tell the pod agent (gem afferent channel) this
        worker finished-and-vanished, so it can relaunch it (typed / new image)."""
        body = (f"pool worker VANISHED — endpoint={self._endpoint} pool={self._pool_id} "
                f"box_type={BOX_TYPE} consumer={self._consumer}. Finished its in-flight task "
                f"and exited(0). Reason: {reason or '(none)'}. Relaunch when ready.")
        if _svc_post_event is None:
            return
        try:
            _svc_post_event(GEM_EVENT_URL, source="pool-worker", severity="notice", body=body, timeout=8)
        except Exception as exc:
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker/{self._consumer}] gem vanish-event failed: {exc}")

    def _guarded(self, fn, what: str, fatal: bool = True):
        """Wrap a daemon-thread target so an unhandled fault cannot vanish quietly.

        A daemon thread that raises simply STOPS: the process keeps running, minus that
        thread. For the heartbeat that is the worst possible outcome — the worker looks alive
        to itself, stops refreshing its registration, and is false-evicted by pool-service
        while still holding a task. Nothing in the exit code, the container state or `docker
        ps` shows it. That is why `fatal=True` here converts a dead heartbeat into an honest
        NON-ZERO process exit: a crashed worker a supervisor can restart beats a zombie one
        nobody can see.

        The config-refresh thread is `fatal=False` — it is best-effort by design (its own
        docstring: "never break boot"), so losing it degrades tuning, not liveness. It still
        logs the crash marker so the loss is visible instead of silent.

        Wrapping the TARGET rather than the loop body is deliberate: it leaves the hot loop
        untouched (no re-indentation of a metronome whose timing is load-bearing) and catches
        faults from anywhere inside it, including the unguarded `_register` call at the top of
        the beat — which is exactly where a redis socket timeout landed on 2026-07-26.
        """
        import functools

        @functools.wraps(fn)
        def _runner():
            try:
                fn()
            except BaseException as exc:                 # noqa: BLE001 — including SystemExit
                tb = traceback.format_exc()
                if fatal:
                    self._die(f"{what} thread died: {type(exc).__name__}: {exc}", tb)
                else:
                    syslog.syslog(syslog.LOG_ERR,
                                  f"[pool_worker/{self._consumer}] {self.CRASH_MARKER} "
                                  f"non-fatal: {what} thread died: {type(exc).__name__}: {exc} "
                                  f"— continuing without it")
                    for line in tb.strip().splitlines()[-8:]:
                        syslog.syslog(syslog.LOG_ERR, f"[pool_worker/{self._consumer}] {line}")
        return _runner

    def _post_gem_crash_event(self, reason: str, tb: str = "") -> None:
        """Counterpart to the vanish event, and deliberately a DIFFERENT one.

        A vanish is planned and reads `notice`; a crash is unplanned and reads `warning`, so
        gem routes it to the event-solver rather than filing it as routine (gem.py: severity
        >= warning -> ganglia). Carries the last few traceback lines because the exit code
        says only THAT it broke — the operator's point on 2026-07-27 was that a code alone
        cannot say WHAT broke, and the answer to that lives in the logs.
        """
        tail = " | ".join(tb.strip().splitlines()[-3:]) if tb else ""
        body = (f"pool worker CRASHED ({self.CRASH_MARKER}) — endpoint={self._endpoint} "
                f"pool={self._pool_id} box_type={BOX_TYPE} consumer={self._consumer}. "
                f"Exited NON-ZERO after an unhandled fault; it did NOT drain. "
                f"Reason: {reason or '(none)'}."
                + (f" Tail: {tail}" if tail else "")
                + " If restart=on-failure the supervisor should bring it back; if it does not,"
                  " the queue for this pool is standing still.")
        if _svc_post_event is None:
            return
        try:
            _svc_post_event(GEM_EVENT_URL, source="pool-worker", severity="warning", body=body, timeout=8)
        except Exception as exc:
            syslog.syslog(syslog.LOG_WARNING,
                          f"[pool_worker/{self._consumer}] gem crash-event failed: {exc}")

    def _post_gem_heartbeat_alarm(self, fails: int) -> None:
        """Fire a gem afferent event when the worker can't reach pool-service for several
        beats (operator 2026-06-19): it's ALIVE but will be (falsely) evicted within
        PINGBACK_TIMEOUT, so the agent should see it live. Best-effort, never blocks the beat."""
        body = (f"pool worker HEARTBEAT FAILING — endpoint={self._endpoint} pool={self._pool_id} "
                f"consumer={self._consumer}: {fails} consecutive _register failures to pool-service. "
                f"Worker is alive but faces false eviction within PINGBACK_TIMEOUT. Check PS reachability.")
        if _svc_post_event is None:
            return
        try:
            _svc_post_event(GEM_EVENT_URL, source="pool-worker", severity="warning", body=body, timeout=5)
        except Exception:
            pass

    def _heartbeat_loop(self):
        # raise THIS thread's priority so a CPU-bound Execute() can never starve the beat
        # (operator 2026-06-23 — the false-eviction-of-a-busy-worker root fix). Per-thread on
        # Linux (the pthread is its own task); needs CAP_SYS_NICE (launch scripts add it).
        try:
            os.setpriority(os.PRIO_PROCESS, threading.get_native_id(), HEARTBEAT_NICE)
        except (OSError, AttributeError, PermissionError) as exc:
            syslog.syslog(syslog.LOG_INFO,
                          f"[pool_worker/{self._consumer}] heartbeat nice {HEARTBEAT_NICE} "
                          f"not applied ({exc}) — running at default priority")
        interval = max(1, HEARTBEAT_INTERVAL)
        hb_fails = 0
        while self._running:
            # FIXED metronome — the interval is the launch-time HEARTBEAT_INTERVAL (set above) and
            # NEVER changes: not live-tuned, not dynamic. A load-elastic beat smears spikes flat
            # (operator, battle-tested) and a too-slow beat false-evicts a live worker. So this loop
            # does NOT re-read the interval; PS_HEARTBEAT_INTERVAL is in the config-safety set.
            time.sleep(interval)
            if not self._running:
                break
            # LIVENESS REFRESH FIRST + UNCONDITIONAL (operator + audit 2026-06-19 — the
            # worker-side mirror of the pool_service fix). A running worker IS alive; refresh
            # our PS registration at the TOP of every beat, BEFORE the drain-check (a Redis
            # GET that can stall up to socket_timeout), so liveness is never gated behind a
            # secondary-store read. We report our current state; a pending drain is caught
            # just below / by the post-task path — a one-beat lag is harmless.
            if self._register(self.GetState()):
                hb_fails = 0
            else:
                hb_fails += 1
                syslog.syslog(syslog.LOG_WARNING,
                              f"[pool_worker/{self._consumer}] heartbeat FAILED to reach "
                              f"pool-service ({hb_fails} consecutive) — at risk of false eviction")
                if hb_fails == 3:
                    self._post_gem_heartbeat_alarm(hb_fails)
            # DRAIN CHECK SECOND — best-effort; a stall here no longer costs liveness.
            drain_info = None
            if not self._draining:
                try:
                    drain_info = self._check_drain()
                except Exception:
                    drain_info = None              # a stalled drain-read must not skip the beat
            if self._draining or drain_info is not None:
                self._draining = True
                if drain_info and drain_info.get("checkpoint"):
                    self._drain_at_checkpoint = True   # in-flight Execute stops at its next checkpoint
                if self.GetState() == STATE_FREE:
                    self._vanish()                 # idle or current task done -> exit(0)
                else:
                    self._register(STATE_FINISHING_AND_SHUTTING_DOWN)
                continue                           # drain WINS over freeze — don't also evaluate freeze
            # FREEZE CHECK THIRD — best-effort, reversible. Only reached when NOT draining
            # (drain is an irreversible exit; freeze is a reversible hold). The next
            # _register at the TOP of the loop already reported our state, so freezing here
            # means the FOLLOWING beat reports FROZEN -> the ping stays fresh -> we are never
            # reaped. A frozen worker that thaws flips straight back to FREE, model resident.
            try:
                frozen_now = self._check_freeze()
            except Exception:
                frozen_now = self._frozen          # a stalled freeze-read holds the prior view
            if frozen_now:
                # Enter (or stay in) the hold. Do NOT interrupt an in-flight task: only flip
                # the visible state to FROZEN when we're FREE or already FROZEN — a BUSY
                # worker keeps running and the post-task cleanup settles it into FROZEN.
                self._frozen = True
                if self.GetState() in (STATE_FREE, STATE_FROZEN):
                    self._set_state(STATE_FROZEN)
            elif self._frozen:
                # THAW: the freeze key is gone. Drop the hold and become dispatchable again.
                # The resident model is untouched — instantly available.
                self._frozen = False
                if self.GetState() == STATE_FROZEN:
                    self._set_state(STATE_FREE)
                syslog.syslog(syslog.LOG_INFO,
                              f"[pool_worker/{self._consumer}] thawed — FROZEN -> FREE (model resident)")

    def _shutdown(self):
        self._running = False
        self.on_drain()
        self._unregister()
        gc.disable()
        gc.collect(2)
        gc.enable()
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] shutdown complete")

    # ── Signal handlers ────────────────────────────────────────────────────────

    def _on_drain(self, sig, _):
        """SIGTERM / SIGINT — finish current task then exit cleanly."""
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] signal {sig} — draining")
        with self._state_lock:
            self._state = STATE_FINISHING_AND_SHUTTING_DOWN
        self._register(STATE_FINISHING_AND_SHUTTING_DOWN)
        # Stop HTTP server — no more tasks accepted
        threading.Thread(target=self._server.shutdown, daemon=True).start()

    def _on_reload(self, sig, _):
        """SIGHUP — reconnect Redis, re-fetch the live config overlay, re-register. Does not
        interrupt a running task. Runs on the MAIN thread (signal context), NOT the heartbeat
        thread, so the bounded config fetch here can never delay a beat (the beat is on its own
        thread). _refresh_config is fully wrapped (never raises), so a config-service blip in a
        SIGHUP can never crash the handler."""
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] SIGHUP — reloading")
        try:
            self._redis.ping()
        except Exception:
            self._redis = _redis_mod.from_url(
                _require_redis_url(), decode_responses=True,
                socket_connect_timeout=5, socket_timeout=60,
                retry_on_timeout=True)
        self._refresh_config(reason="sighup")
        self._register(self.GetState())

    def _on_gc(self, sig, _):
        """SIGUSR1 — force full GC and log memory stats."""
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] SIGUSR1 — forced GC")
        gc.collect(2)
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] GC counts: {gc.get_count()} "
                      f"thresholds: {gc.get_threshold()}")

    def _on_dump(self, sig, _):
        """SIGUSR2 — dump current state to syslog."""
        state = self.GetState()
        syslog.syslog(syslog.LOG_INFO,
                      f"[pool_worker/{self._consumer}] SIGUSR2 dump: "
                      f"state={_STATE_NAMES[state]} "
                      f"endpoint={self._endpoint} "
                      f"pool={self._pool_id} "
                      f"consumer={self._consumer} "
                      f"running={self._running}")
