""" Gateway runner - entry point for messaging platform integrations. This module provides: - start_gateway(): Start all configured platform adapters - GatewayRunner: Main class managing the gateway lifecycle Usage: # Start the gateway python -m gateway.run # Or from CLI python cli.py --gateway """ import asyncio import dataclasses import inspect import json import logging import os import re import shlex import sys import signal import tempfile import threading import time from collections import OrderedDict from contextvars import copy_context from pathlib import Path from datetime import datetime from typing import Dict, Optional, Any, List, Union # account_usage imports the OpenAI SDK chain (~230 ms). Only needed by # /usage; we still import it at module top in the gateway because test # patches (tests/gateway/test_usage_command.py) target # `gateway.run.fetch_account_usage` as a module-level attribute. The # gateway is a long-running daemon, so its boot cost matters less than # preserving the established test-patch surface. from agent.account_usage import fetch_account_usage, render_account_usage_lines from hermes_cli.config import cfg_get # --- Agent cache tuning --------------------------------------------------- # Bounds the per-session AIAgent cache to prevent unbounded growth in # long-lived gateways (each AIAgent holds LLM clients, tool schemas, # memory providers, etc.). LRU order + idle TTL eviction are enforced # from _enforce_agent_cache_cap() and _session_expiry_watcher() below. _AGENT_CACHE_MAX_SIZE = 128 _AGENT_CACHE_IDLE_TTL_SECS = 3600.0 # evict agents idle for >1h _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT = 30.0 # Only auto-continue interrupted gateway turns while the interruption is fresh. # Stale tool-tail/resume markers can otherwise revive an unrelated old task # after a gateway restart when the user's next message starts new work. # # The freshness signal is the timestamp of the last transcript row, which # ``hermes_state.get_messages`` carries on every persisted message. This # handles the two auto-continue cases uniformly: # * resume_pending (gateway restart/shutdown watchdog marked the session) # * tool-tail (last persisted message is a tool result the agent # never got to reply to) # In both cases "when did we last do anything on this transcript" is the # correct freshness question, so one signal replaces two divergent ones. # # Default window: 1 hour. This comfortably covers ``agent.gateway_timeout`` # (30 min default) plus runtime slack — a legitimate long-running turn that # gets interrupted near its timeout boundary and is resumed shortly after # is still classified fresh. Override via # ``config.yaml`` ``agent.gateway_auto_continue_freshness``. _AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT = 60 * 60 # --- Stale-code self-check ------------------------------------------------ # Long-running gateway processes that survive an ``hermes update`` keep the # old ``hermes_cli.config`` (and friends) cached in ``sys.modules``. When # the updated tool files on disk then try to ``from hermes_cli.config # import cfg_get`` (added in PR #17304), the import resolves against the # already-loaded stale module object and raises ``ImportError`` — see # Issue #17648. Rather than papering over the import failure site-by-site # in every tool file, detect the stale state centrally and auto-restart # so the gateway reloads with fresh code. The sentinel files below are # the canonical repo-level markers that every update touches; if any is # newer than the gateway's boot time, we know the running process is out # of date. _STALE_CODE_SENTINELS: tuple[str, ...] = ( "hermes_cli/config.py", "hermes_cli/__init__.py", "run_agent.py", "gateway/run.py", "pyproject.toml", ) def _compute_repo_mtime(repo_root: Path) -> float: """Return the newest mtime across the stale-code sentinel files. Missing files are ignored (they may not exist on older checkouts). Returns 0.0 if no sentinel file is readable — treat that as "can't tell", which downstream callers interpret as "not stale" to avoid false-positive restart loops. """ newest = 0.0 for rel in _STALE_CODE_SENTINELS: try: st = (repo_root / rel).stat() except (OSError, FileNotFoundError): continue if st.st_mtime > newest: newest = st.st_mtime return newest def _coerce_gateway_timestamp(value: Any) -> Optional[float]: """Best-effort conversion of stored gateway timestamps to epoch seconds. Missing/unparseable timestamps return None so legacy transcripts keep the historical auto-continue behaviour instead of being silently dropped. Accepts: datetime, epoch seconds (int/float), epoch milliseconds (when the magnitude exceeds year-2286), ISO-8601 strings (with or without a trailing ``Z``), and numeric strings. """ if value is None: return None if isinstance(value, datetime): return value.timestamp() if isinstance(value, bool): # bool is a subclass of int — skip it return None if isinstance(value, (int, float)): # Some platform events use milliseconds; Hermes state rows use seconds. return float(value) / 1000.0 if float(value) > 10_000_000_000 else float(value) if isinstance(value, str): text = value.strip() if not text: return None try: numeric = float(text) return numeric / 1000.0 if numeric > 10_000_000_000 else numeric except ValueError: pass try: return datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp() except ValueError: return None return None def _auto_continue_freshness_window() -> float: """Return the configured auto-continue freshness window in seconds. Reads ``HERMES_AUTO_CONTINUE_FRESHNESS`` (bridged from ``config.yaml`` ``agent.gateway_auto_continue_freshness`` at gateway startup, same pattern as ``HERMES_AGENT_TIMEOUT``). Falls back to the module default when unset or malformed. Non-positive values disable the freshness gate (restores the pre-fix "always fresh" behaviour for users who want to opt out). """ raw = os.environ.get("HERMES_AUTO_CONTINUE_FRESHNESS") if raw is None or raw == "": return float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT) try: return float(raw) except (TypeError, ValueError): return float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT) def _float_env(name: str, default: float) -> float: """Read an env var as float, falling back to ``default`` on typos/empty. A misconfigured env var (e.g. ``HERMES_AGENT_TIMEOUT=abc``) must not crash the gateway or an agent turn. Unset/empty also falls back. """ raw = os.environ.get(name) if raw is None or raw == "": return float(default) try: return float(raw) except (TypeError, ValueError): return float(default) def _is_fresh_gateway_interruption( value: Any, *, now: Optional[float] = None, window_secs: Optional[float] = None, ) -> bool: """Return True when an interruption marker is fresh enough to auto-continue. Unknown timestamps are treated as fresh for backward compatibility with legacy transcripts (pre-dating timestamp persistence) and with in-memory test scaffolding that constructs history entries without timestamps. A non-positive ``window_secs`` disables the gate (always fresh), which restores the pre-fix behaviour for users who opt out via config. """ window = ( float(window_secs) if window_secs is not None else float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT) ) if window <= 0: return True timestamp = _coerce_gateway_timestamp(value) if timestamp is None: return True current = time.time() if now is None else now return current - timestamp <= window def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any: """Return the ``timestamp`` of the last usable transcript row, if any. Skips metadata-only rows (``session_meta``, system injections) that are dropped before being handed to the agent. Returns ``None`` when no usable row carries a timestamp — callers should treat that as "fresh" for backward compatibility. """ if not history: return None for msg in reversed(history): if not isinstance(msg, dict): continue role = msg.get("role") if not role or role in ("session_meta", "system"): continue ts = msg.get("timestamp") if ts is not None: return ts # First non-meta row without a timestamp — legacy transcript row. # Returning None lets the caller fall through to the legacy-fresh path. return None return None # --------------------------------------------------------------------------- # SSL certificate auto-detection for NixOS and other non-standard systems. # Must run BEFORE any HTTP library (discord, aiohttp, etc.) is imported. # --------------------------------------------------------------------------- def _ensure_ssl_certs() -> None: """Set SSL_CERT_FILE if the system doesn't expose CA certs to Python.""" if "SSL_CERT_FILE" in os.environ: return # user already configured it import ssl # 1. Python's compiled-in defaults paths = ssl.get_default_verify_paths() for candidate in (paths.cafile, paths.openssl_cafile): if candidate and os.path.exists(candidate): os.environ["SSL_CERT_FILE"] = candidate return # 2. certifi (ships its own Mozilla bundle) try: import certifi os.environ["SSL_CERT_FILE"] = certifi.where() return except ImportError: pass # 3. Common distro / macOS locations for candidate in ( "/etc/ssl/certs/ca-certificates.crt", # Debian/Ubuntu/Gentoo "/etc/pki/tls/certs/ca-bundle.crt", # RHEL/CentOS 7 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # RHEL/CentOS 8+ "/etc/ssl/ca-bundle.pem", # SUSE/OpenSUSE "/etc/ssl/cert.pem", # Alpine / macOS "/etc/pki/tls/cert.pem", # Fedora "/usr/local/etc/openssl@1.1/cert.pem", # macOS Homebrew Intel "/opt/homebrew/etc/openssl@1.1/cert.pem", # macOS Homebrew ARM ): if os.path.exists(candidate): os.environ["SSL_CERT_FILE"] = candidate return def _home_target_env_var(platform_name: str) -> str: """Return the configured home-target env var for a platform.""" from cron.scheduler import _HOME_TARGET_ENV_VARS return _HOME_TARGET_ENV_VARS.get( platform_name.lower(), f"{platform_name.upper()}_HOME_CHANNEL", ) _ensure_ssl_certs() # Add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) # Resolve Hermes home directory (respects HERMES_HOME override) from hermes_constants import get_hermes_home from utils import atomic_json_write, atomic_yaml_write, base_url_host_matches, is_truthy_value _hermes_home = get_hermes_home() # Load environment variables from ~/.hermes/.env first. # User-managed env files should override stale shell exports on restart. from dotenv import load_dotenv # backward-compat for tests that monkeypatch this symbol from hermes_cli.env_loader import load_hermes_dotenv _env_path = _hermes_home / '.env' load_hermes_dotenv(hermes_home=_hermes_home, project_env=Path(__file__).resolve().parents[1] / '.env') _DOCKER_VOLUME_SPEC_RE = re.compile(r"^(?P.+):(?P/[^:]+?)(?::(?P[^:]+))?$") _DOCKER_MEDIA_OUTPUT_CONTAINER_PATHS = {"/output", "/outputs"} # Bridge config.yaml values into the environment so os.getenv() picks them up. # config.yaml is authoritative for terminal settings — overrides .env. _config_path = _hermes_home / 'config.yaml' if _config_path.exists(): try: import yaml as _yaml with open(_config_path, encoding="utf-8") as _f: _cfg = _yaml.safe_load(_f) or {} # Expand ${ENV_VAR} references before bridging to env vars. from hermes_cli.config import _expand_env_vars _cfg = _expand_env_vars(_cfg) # Top-level simple values (fallback only — don't override .env) for _key, _val in _cfg.items(): if isinstance(_val, (str, int, float, bool)) and _key not in os.environ: os.environ[_key] = str(_val) # Terminal config is nested — bridge to TERMINAL_* env vars. # config.yaml overrides .env for these since it's the documented config path. _terminal_cfg = _cfg.get("terminal", {}) if _terminal_cfg and isinstance(_terminal_cfg, dict): _terminal_env_map = { "backend": "TERMINAL_ENV", "cwd": "TERMINAL_CWD", "timeout": "TERMINAL_TIMEOUT", "lifetime_seconds": "TERMINAL_LIFETIME_SECONDS", "docker_image": "TERMINAL_DOCKER_IMAGE", "docker_forward_env": "TERMINAL_DOCKER_FORWARD_ENV", "singularity_image": "TERMINAL_SINGULARITY_IMAGE", "modal_image": "TERMINAL_MODAL_IMAGE", "daytona_image": "TERMINAL_DAYTONA_IMAGE", "vercel_runtime": "TERMINAL_VERCEL_RUNTIME", "ssh_host": "TERMINAL_SSH_HOST", "ssh_user": "TERMINAL_SSH_USER", "ssh_port": "TERMINAL_SSH_PORT", "ssh_key": "TERMINAL_SSH_KEY", "container_cpu": "TERMINAL_CONTAINER_CPU", "container_memory": "TERMINAL_CONTAINER_MEMORY", "container_disk": "TERMINAL_CONTAINER_DISK", "container_persistent": "TERMINAL_CONTAINER_PERSISTENT", "docker_volumes": "TERMINAL_DOCKER_VOLUMES", "docker_mount_cwd_to_workspace": "TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "docker_run_as_host_user": "TERMINAL_DOCKER_RUN_AS_HOST_USER", "sandbox_dir": "TERMINAL_SANDBOX_DIR", "persistent_shell": "TERMINAL_PERSISTENT_SHELL", } for _cfg_key, _env_var in _terminal_env_map.items(): if _cfg_key in _terminal_cfg: _val = _terminal_cfg[_cfg_key] # Skip cwd placeholder values (".", "auto", "cwd") — the # gateway resolves these to Path.home() later (line ~255). # Writing the raw placeholder here would just be noise. # Only bridge explicit absolute paths from config.yaml. if _cfg_key == "cwd" and str(_val) in (".", "auto", "cwd"): continue # Expand shell tilde in cwd so subprocess.Popen never # receives a literal "~/" which the kernel rejects. if _cfg_key == "cwd" and isinstance(_val, str): _val = os.path.expanduser(_val) if isinstance(_val, list): os.environ[_env_var] = json.dumps(_val) else: os.environ[_env_var] = str(_val) # Compression config is read directly from config.yaml by run_agent.py # and auxiliary_client.py — no env var bridging needed. # Auxiliary model/direct-endpoint overrides (vision, web_extract). # Each task has provider/model/base_url/api_key; bridge non-default values to env vars. _auxiliary_cfg = _cfg.get("auxiliary", {}) if _auxiliary_cfg and isinstance(_auxiliary_cfg, dict): _aux_task_env = { "vision": { "provider": "AUXILIARY_VISION_PROVIDER", "model": "AUXILIARY_VISION_MODEL", "base_url": "AUXILIARY_VISION_BASE_URL", "api_key": "AUXILIARY_VISION_API_KEY", }, "web_extract": { "provider": "AUXILIARY_WEB_EXTRACT_PROVIDER", "model": "AUXILIARY_WEB_EXTRACT_MODEL", "base_url": "AUXILIARY_WEB_EXTRACT_BASE_URL", "api_key": "AUXILIARY_WEB_EXTRACT_API_KEY", }, "approval": { "provider": "AUXILIARY_APPROVAL_PROVIDER", "model": "AUXILIARY_APPROVAL_MODEL", "base_url": "AUXILIARY_APPROVAL_BASE_URL", "api_key": "AUXILIARY_APPROVAL_API_KEY", }, } for _task_key, _env_map in _aux_task_env.items(): _task_cfg = _auxiliary_cfg.get(_task_key, {}) if not isinstance(_task_cfg, dict): continue _prov = str(_task_cfg.get("provider", "")).strip() _model = str(_task_cfg.get("model", "")).strip() _base_url = str(_task_cfg.get("base_url", "")).strip() _api_key = str(_task_cfg.get("api_key", "")).strip() if _prov and _prov != "auto": os.environ[_env_map["provider"]] = _prov if _model: os.environ[_env_map["model"]] = _model if _base_url: os.environ[_env_map["base_url"]] = _base_url if _api_key: os.environ[_env_map["api_key"]] = _api_key # config.yaml is the documented, authoritative source for these # settings — it unconditionally wins over .env values. Previously # the guards below read `if X not in os.environ` and let stale # .env entries (e.g. HERMES_MAX_ITERATIONS=60 written by an old # `hermes setup` run) silently shadow the user's current config. # See PR #18413 / the 60-vs-500 max_turns incident. _agent_cfg = _cfg.get("agent", {}) if _agent_cfg and isinstance(_agent_cfg, dict): if "max_turns" in _agent_cfg: os.environ["HERMES_MAX_ITERATIONS"] = str(_agent_cfg["max_turns"]) if "gateway_timeout" in _agent_cfg: os.environ["HERMES_AGENT_TIMEOUT"] = str(_agent_cfg["gateway_timeout"]) if "gateway_timeout_warning" in _agent_cfg: os.environ["HERMES_AGENT_TIMEOUT_WARNING"] = str(_agent_cfg["gateway_timeout_warning"]) if "gateway_notify_interval" in _agent_cfg: os.environ["HERMES_AGENT_NOTIFY_INTERVAL"] = str(_agent_cfg["gateway_notify_interval"]) if "restart_drain_timeout" in _agent_cfg: os.environ["HERMES_RESTART_DRAIN_TIMEOUT"] = str(_agent_cfg["restart_drain_timeout"]) if "gateway_auto_continue_freshness" in _agent_cfg: os.environ["HERMES_AUTO_CONTINUE_FRESHNESS"] = str( _agent_cfg["gateway_auto_continue_freshness"] ) _display_cfg = _cfg.get("display", {}) if _display_cfg and isinstance(_display_cfg, dict): if "busy_input_mode" in _display_cfg: os.environ["HERMES_GATEWAY_BUSY_INPUT_MODE"] = str(_display_cfg["busy_input_mode"]) if "busy_ack_enabled" in _display_cfg: os.environ["HERMES_GATEWAY_BUSY_ACK_ENABLED"] = str(_display_cfg["busy_ack_enabled"]) # Timezone: bridge config.yaml → HERMES_TIMEZONE env var. _tz_cfg = _cfg.get("timezone", "") if _tz_cfg and isinstance(_tz_cfg, str): os.environ["HERMES_TIMEZONE"] = _tz_cfg.strip() # Security settings _security_cfg = _cfg.get("security", {}) if isinstance(_security_cfg, dict): _redact = _security_cfg.get("redact_secrets") if _redact is not None: os.environ["HERMES_REDACT_SECRETS"] = str(_redact).lower() except Exception as _bridge_err: # Previously this was silent (`except Exception: pass`), which # hid partial bridge failures and let .env defaults shadow # config.yaml values — users observed max_turns=500 in config # but a 60-iteration cap in practice. Surface the failure to # stderr so operators see it even though `logger` is not yet # initialized at module-import time (logger is defined further # down this module). print( f" Warning: config.yaml → env bridge failed: " f"{type(_bridge_err).__name__}: {_bridge_err}", file=sys.stderr, ) print( " Gateway will fall back to .env values, which may not match " "your current config.yaml. Run `hermes doctor` to investigate.", file=sys.stderr, ) # Apply IPv4 preference if configured (before any HTTP clients are created). try: from hermes_constants import apply_ipv4_preference _network_cfg = (_cfg if '_cfg' in dir() else {}).get("network", {}) if isinstance(_network_cfg, dict) and _network_cfg.get("force_ipv4"): apply_ipv4_preference(force=True) except Exception: pass # Validate config structure early — log warnings so gateway operators see problems try: from hermes_cli.config import print_config_warnings print_config_warnings() except Exception: pass # Warn if user has deprecated MESSAGING_CWD / TERMINAL_CWD in .env try: from hermes_cli.config import warn_deprecated_cwd_env_vars warn_deprecated_cwd_env_vars() except Exception: pass # Gateway runs in quiet mode - suppress debug output and use cwd directly (no temp dirs) os.environ["HERMES_QUIET"] = "1" # Enable interactive exec approval for dangerous commands on messaging platforms os.environ["HERMES_EXEC_ASK"] = "1" # Set terminal working directory for messaging platforms. # config.yaml terminal.cwd is the canonical source (bridged to TERMINAL_CWD # by the config bridge above). When it's unset or a placeholder, default # to home directory. MESSAGING_CWD is accepted as a backward-compat # fallback (deprecated — the warning above tells users to migrate). _configured_cwd = os.environ.get("TERMINAL_CWD", "") if not _configured_cwd or _configured_cwd in (".", "auto", "cwd"): _fallback = os.getenv("MESSAGING_CWD") or str(Path.home()) os.environ["TERMINAL_CWD"] = _fallback from gateway.config import ( Platform, _BUILTIN_PLATFORM_VALUES, GatewayConfig, load_gateway_config, ) from gateway.session import ( SessionStore, SessionSource, SessionContext, build_session_context, build_session_context_prompt, build_session_key, is_shared_multi_user_session, ) from gateway.delivery import DeliveryRouter from gateway.platforms.base import ( BasePlatformAdapter, EphemeralReply, MessageEvent, MessageType, merge_pending_message_event, ) from gateway.restart import ( DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, GATEWAY_SERVICE_RESTART_EXIT_CODE, parse_restart_drain_timeout, ) from gateway.whatsapp_identity import ( canonical_whatsapp_identifier as _canonical_whatsapp_identifier, # noqa: F401 expand_whatsapp_aliases as _expand_whatsapp_auth_aliases, normalize_whatsapp_identifier as _normalize_whatsapp_identifier, ) logger = logging.getLogger(__name__) # Sentinel placed into _running_agents immediately when a session starts # processing, *before* any await. Prevents a second message for the same # session from bypassing the "already running" guard during the async gap # between the guard check and actual agent creation. _AGENT_PENDING_SENTINEL = object() def _resolve_runtime_agent_kwargs() -> dict: """Resolve provider credentials for gateway-created AIAgent instances. If the primary provider fails with an authentication error, attempt to resolve credentials using the fallback provider chain from config.yaml before giving up. """ from hermes_cli.runtime_provider import ( resolve_runtime_provider, format_runtime_provider_error, ) from hermes_cli.auth import AuthError try: runtime = resolve_runtime_provider( requested=os.getenv("HERMES_INFERENCE_PROVIDER"), ) except AuthError as auth_exc: # Primary provider auth failed (expired token, revoked key, etc.). # Try the fallback provider chain before raising. logger.warning("Primary provider auth failed: %s — trying fallback", auth_exc) fb_config = _try_resolve_fallback_provider() if fb_config is not None: return fb_config raise RuntimeError(format_runtime_provider_error(auth_exc)) from auth_exc except Exception as exc: raise RuntimeError(format_runtime_provider_error(exc)) from exc return { "api_key": runtime.get("api_key"), "base_url": runtime.get("base_url"), "provider": runtime.get("provider"), "api_mode": runtime.get("api_mode"), "command": runtime.get("command"), "args": list(runtime.get("args") or []), "credential_pool": runtime.get("credential_pool"), } def _try_resolve_fallback_provider() -> dict | None: """Attempt to resolve credentials from the fallback_model/fallback_providers config.""" from hermes_cli.runtime_provider import resolve_runtime_provider try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if not cfg_path.exists(): return None with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} fb = cfg.get("fallback_providers") or cfg.get("fallback_model") if not fb: return None # Normalize to list fb_list = fb if isinstance(fb, list) else [fb] for entry in fb_list: if not isinstance(entry, dict): continue try: runtime = resolve_runtime_provider( requested=entry.get("provider"), explicit_base_url=entry.get("base_url"), explicit_api_key=entry.get("api_key"), ) logger.info("Fallback provider resolved: %s", runtime.get("provider")) return { "api_key": runtime.get("api_key"), "base_url": runtime.get("base_url"), "provider": runtime.get("provider"), "api_mode": runtime.get("api_mode"), "command": runtime.get("command"), "args": list(runtime.get("args") or []), "credential_pool": runtime.get("credential_pool"), } except Exception as fb_exc: logger.debug("Fallback entry %s failed: %s", entry.get("provider"), fb_exc) continue except Exception: pass return None def _build_media_placeholder(event) -> str: """Build a text placeholder for media-only events so they aren't dropped. When a photo/document is queued during active processing and later dequeued, only .text is extracted. If the event has no caption, the media would be silently lost. This builds a placeholder that the vision enrichment pipeline will replace with a real description. """ parts = [] media_urls = getattr(event, "media_urls", None) or [] media_types = getattr(event, "media_types", None) or [] for i, url in enumerate(media_urls): mtype = media_types[i] if i < len(media_types) else "" if mtype.startswith("image/") or getattr(event, "message_type", None) == MessageType.PHOTO: parts.append(f"[User sent an image: {url}]") elif mtype.startswith("audio/"): parts.append(f"[User sent audio: {url}]") else: parts.append(f"[User sent a file: {url}]") return "\n".join(parts) def _dequeue_pending_event(adapter, session_key: str) -> MessageEvent | None: """Consume and return the full pending event for a session. Queued follow-ups must preserve their media metadata so they can re-enter the normal image/STT/document preprocessing path instead of being reduced to a placeholder string. """ return adapter.get_pending_message(session_key) _INTERRUPT_REASON_STOP = "Stop requested" _INTERRUPT_REASON_RESET = "Session reset requested" _INTERRUPT_REASON_TIMEOUT = "Execution timed out (inactivity)" _INTERRUPT_REASON_SSE_DISCONNECT = "SSE client disconnected" _INTERRUPT_REASON_GATEWAY_SHUTDOWN = "Gateway shutting down" _INTERRUPT_REASON_GATEWAY_RESTART = "Gateway restarting" _CONTROL_INTERRUPT_MESSAGES = frozenset( { _INTERRUPT_REASON_STOP.lower(), _INTERRUPT_REASON_RESET.lower(), _INTERRUPT_REASON_TIMEOUT.lower(), _INTERRUPT_REASON_SSE_DISCONNECT.lower(), _INTERRUPT_REASON_GATEWAY_SHUTDOWN.lower(), _INTERRUPT_REASON_GATEWAY_RESTART.lower(), } ) def _is_control_interrupt_message(message: Optional[str]) -> bool: """Return True when an interrupt message is internal control flow.""" if not message: return False normalized = " ".join(str(message).strip().split()).lower() return normalized in _CONTROL_INTERRUPT_MESSAGES def _skill_slug_from_frontmatter(skill_md: Path) -> tuple[str | None, str | None]: """Derive the /command slug and declared frontmatter name from a SKILL.md. Matches the exact normalization used by :func:`agent.skill_commands.scan_skill_commands` so the slug here is the same string a user types after the leading ``/`` (e.g. a skill with frontmatter ``name: Stable Diffusion Image Generation`` resolves to ``stable-diffusion-image-generation`` — NOT the parent directory name, which is commonly shorter/different, e.g. ``stable-diffusion``). Using the directory name silently broke :func:`_check_unavailable_skill` for every skill whose directory name drifted from its frontmatter name (19 such skills on a standard install as of 2026-05), causing a generic "unknown command" response where a "disabled — enable with …" or "not installed — install with …" hint was expected. Returns ``(slug, declared_name)`` or ``(None, None)`` when the file can't be read or lacks a ``name:`` in its frontmatter. """ try: content = skill_md.read_text(encoding="utf-8", errors="replace") except Exception: return None, None if not content.startswith("---"): return None, None end = content.find("\n---", 3) if end < 0: return None, None declared_name: str | None = None for line in content[3:end].splitlines(): line = line.strip() if line.startswith("name:"): raw = line.split(":", 1)[1].strip() # Strip YAML quote wrappers if present if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in ('"', "'"): raw = raw[1:-1] declared_name = raw.strip() break if not declared_name: return None, None slug = declared_name.lower().replace(" ", "-").replace("_", "-") # Mirror _SKILL_INVALID_CHARS and _SKILL_MULTI_HYPHEN from skill_commands import re as _re slug = _re.sub(r"[^a-z0-9-]", "", slug) slug = _re.sub(r"-{2,}", "-", slug).strip("-") if not slug: return None, declared_name return slug, declared_name def _check_unavailable_skill(command_name: str) -> str | None: """Check if a command matches a known-but-inactive skill. Returns a helpful message if the skill exists but is disabled or only available as an optional install. Returns None if no match found. The slug for each on-disk skill is derived from its frontmatter ``name:`` (via :func:`_skill_slug_from_frontmatter`), NOT from its containing directory name — because the two can differ (e.g. directory ``stable-diffusion`` + frontmatter ``Stable Diffusion Image Generation`` yields slug ``stable-diffusion-image-generation``). Matching on directory name would miss that slug entirely and fall through to the generic "unknown command" path. """ # Normalize: command uses hyphens, skill names may use hyphens or underscores normalized = command_name.lower().replace("_", "-") try: from tools.skills_tool import _get_disabled_skill_names from agent.skill_utils import get_all_skills_dirs disabled = _get_disabled_skill_names() # Check disabled skills across all dirs (local + external) for skills_dir in get_all_skills_dirs(): if not skills_dir.exists(): continue for skill_md in skills_dir.rglob("SKILL.md"): if any(part in ('.git', '.github', '.hub', '.archive') for part in skill_md.parts): continue slug, declared_name = _skill_slug_from_frontmatter(skill_md) if not slug or not declared_name: continue # disabled is keyed by the declared frontmatter name (what # skills.disabled / skills.platform_disabled store). if slug == normalized and declared_name in disabled: return ( f"The **{command_name}** skill is installed but disabled.\n" f"Enable it with: `hermes skills config`" ) # Check optional skills (shipped with repo but not installed) from hermes_constants import get_optional_skills_dir repo_root = Path(__file__).resolve().parent.parent optional_dir = get_optional_skills_dir(repo_root / "optional-skills") if optional_dir.exists(): for skill_md in optional_dir.rglob("SKILL.md"): slug, _declared = _skill_slug_from_frontmatter(skill_md) if not slug: continue if slug == normalized: # Build install path: official// rel = skill_md.parent.relative_to(optional_dir) parts = list(rel.parts) install_path = f"official/{'/'.join(parts)}" return ( f"The **{command_name}** skill is available but not installed.\n" f"Install it with: `hermes skills install {install_path}`" ) except Exception: pass return None def _platform_config_key(platform: "Platform") -> str: """Map a Platform enum to its config.yaml key (LOCAL→"cli", rest→enum value).""" return "cli" if platform == Platform.LOCAL else platform.value def _load_gateway_config() -> dict: """Load and parse ~/.hermes/config.yaml, returning {} on any error. Uses the module-level ``_hermes_home`` (so tests that monkeypatch it still see their fixture) and shares the mtime-keyed raw-yaml cache from ``hermes_cli.config.read_raw_config`` when the paths match. """ config_path = _hermes_home / 'config.yaml' try: from hermes_cli.config import get_config_path, read_raw_config # Fast path: if _hermes_home agrees with the canonical config # location, reuse the shared cache. Otherwise fall through to a # direct read (keeps test fixtures with a monkeypatched # _hermes_home working). if config_path == get_config_path(): return read_raw_config() except Exception: pass try: if config_path.exists(): import yaml with open(config_path, 'r', encoding='utf-8') as f: return yaml.safe_load(f) or {} except Exception: logger.debug("Could not load gateway config from %s", config_path) return {} def _resolve_gateway_model(config: dict | None = None) -> str: """Read model from config.yaml — single source of truth. Without this, temporary AIAgent instances (e.g. /compress) fall back to the hardcoded default which fails when the active provider is openai-codex. """ cfg = config if config is not None else _load_gateway_config() model_cfg = cfg.get("model", {}) if isinstance(model_cfg, str): return model_cfg elif isinstance(model_cfg, dict): return model_cfg.get("default") or model_cfg.get("model") or "" return "" def _resolve_hermes_bin() -> Optional[list[str]]: """Resolve the Hermes update command as argv parts. Tries in order: 1. ``shutil.which("hermes")`` — standard PATH lookup 2. ``sys.executable -m hermes_cli.main`` — fallback when Hermes is running from a venv/module invocation and the ``hermes`` shim is not on PATH Returns argv parts ready for quoting/joining, or ``None`` if neither works. """ import shutil hermes_bin = shutil.which("hermes") if hermes_bin: return [hermes_bin] try: import importlib.util if importlib.util.find_spec("hermes_cli") is not None: return [sys.executable, "-m", "hermes_cli.main"] except Exception: pass return None def _parse_session_key(session_key: str) -> "dict | None": """Parse a session key into its component parts. Session keys follow the format ``agent:main:{platform}:{chat_type}:{chat_id}[:{extra}...]``. Returns a dict with ``platform``, ``chat_type``, ``chat_id``, and optionally ``thread_id`` keys, or None if the key doesn't match. The 6th element is only returned as ``thread_id`` for chat types where it is unambiguous (``dm`` and ``thread``). For group/channel sessions the suffix may be a user_id (per-user isolation) rather than a thread_id, so we leave ``thread_id`` out to avoid mis-routing. """ parts = session_key.split(":") if len(parts) >= 5 and parts[0] == "agent" and parts[1] == "main": result = { "platform": parts[2], "chat_type": parts[3], "chat_id": parts[4], } if len(parts) > 5 and parts[3] in ("dm", "thread"): result["thread_id"] = parts[5] return result return None def _format_gateway_process_notification(evt: dict) -> "str | None": """Format a watch pattern event from completion_queue into a [IMPORTANT:] message.""" evt_type = evt.get("type", "completion") _sid = evt.get("session_id", "unknown") _cmd = evt.get("command", "unknown") if evt_type == "watch_disabled": return f"[IMPORTANT: {evt.get('message', '')}]" if evt_type == "watch_match": _pat = evt.get("pattern", "?") _out = evt.get("output", "") _sup = evt.get("suppressed", 0) text = ( f"[IMPORTANT: Background process {_sid} matched " f"watch pattern \"{_pat}\".\n" f"Command: {_cmd}\n" f"Matched output:\n{_out}" ) if _sup: text += f"\n({_sup} earlier matches were suppressed by rate limit)" text += "]" return text return None # Module-level weak reference to the active GatewayRunner instance. # Used by tools (e.g. send_message) that need to route through a live # adapter for plugin platforms. Set in GatewayRunner.__init__(). import weakref as _weakref _gateway_runner_ref: _weakref.ref = lambda: None class GatewayRunner: """ Main gateway controller. Manages the lifecycle of all platform adapters and routes messages to/from the agent. """ # Class-level defaults so partial construction in tests doesn't # blow up on attribute access. _running_agents_ts: Dict[str, float] = {} _busy_input_mode: str = "interrupt" _restart_drain_timeout: float = DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT _exit_code: Optional[int] = None _draining: bool = False _restart_requested: bool = False _restart_task_started: bool = False _restart_detached: bool = False _restart_via_service: bool = False _stop_task: Optional[asyncio.Task] = None _session_model_overrides: Dict[str, Dict[str, str]] = {} _session_reasoning_overrides: Dict[str, Dict[str, Any]] = {} # Stale-code self-check defaults (see _detect_stale_code()). Class-level # so tests that construct GatewayRunner via ``object.__new__`` without # running __init__ don't crash when _handle_message reads these. _boot_wall_time: float = 0.0 _boot_repo_mtime: float = 0.0 _stale_code_restart_triggered: bool = False def __init__(self, config: Optional[GatewayConfig] = None): global _gateway_runner_ref self.config = config or load_gateway_config() self.adapters: Dict[Platform, BasePlatformAdapter] = {} self._warn_if_docker_media_delivery_is_risky() _gateway_runner_ref = _weakref.ref(self) # Boot-time snapshot used by the stale-code self-check. Captured # before any work happens so post-update file writes are guaranteed # to have newer mtimes. See _detect_stale_code() / Issue #17648. try: self._boot_wall_time: float = time.time() self._repo_root_for_staleness: Path = Path(__file__).resolve().parent.parent self._boot_repo_mtime: float = _compute_repo_mtime( self._repo_root_for_staleness, ) except Exception: self._boot_wall_time = 0.0 self._repo_root_for_staleness = Path(".") self._boot_repo_mtime = 0.0 self._stale_code_notified: set[str] = set() self._stale_code_restart_triggered: bool = False # Load ephemeral config from config.yaml / env vars. # Both are injected at API-call time only and never persisted. self._prefill_messages = self._load_prefill_messages() self._ephemeral_system_prompt = self._load_ephemeral_system_prompt() self._reasoning_config = self._load_reasoning_config() self._service_tier = self._load_service_tier() self._show_reasoning = self._load_show_reasoning() self._busy_input_mode = self._load_busy_input_mode() self._restart_drain_timeout = self._load_restart_drain_timeout() self._provider_routing = self._load_provider_routing() self._fallback_model = self._load_fallback_model() # Wire process registry into session store for reset protection from tools.process_registry import process_registry self.session_store = SessionStore( self.config.sessions_dir, self.config, has_active_processes_fn=lambda key: process_registry.has_active_for_session(key), ) self.delivery_router = DeliveryRouter(self.config) self._running = False self._shutdown_event = asyncio.Event() self._exit_cleanly = False self._exit_with_failure = False self._exit_reason: Optional[str] = None self._exit_code: Optional[int] = None self._draining = False self._restart_requested = False self._restart_task_started = False self._restart_detached = False self._restart_via_service = False self._stop_task: Optional[asyncio.Task] = None # Track running agents per session for interrupt support # Key: session_key, Value: AIAgent instance self._running_agents: Dict[str, Any] = {} self._running_agents_ts: Dict[str, float] = {} # start timestamp per session self._pending_messages: Dict[str, str] = {} # Queued messages during interrupt # Overflow buffer for explicit /queue commands. The adapter-level # _pending_messages dict is a single slot per session (designed for # "next-turn" follow-ups where repeated sends collapse into one # event). /queue has different semantics: each invocation must # produce its own full agent turn, in FIFO order, with no merging. # When the slot is occupied, additional /queue items land here and # are promoted one-at-a-time after each run's drain. Cleared on # /new and /reset. /model and other mid-session operations # preserve the queue. self._queued_events: Dict[str, List[MessageEvent]] = {} self._pending_native_image_paths_by_session: Dict[str, List[str]] = {} self._busy_ack_ts: Dict[str, float] = {} # last busy-ack timestamp per session (debounce) self._session_run_generation: Dict[str, int] = {} # Cache AIAgent instances per session to preserve prompt caching. # Without this, a new AIAgent is created per message, rebuilding the # system prompt (including memory) every turn — breaking prefix cache # and costing ~10x more on providers with prompt caching (Anthropic). # Key: session_key, Value: (AIAgent, config_signature_str) # # OrderedDict so _enforce_agent_cache_cap() can pop the least-recently- # used entry (move_to_end() on cache hits, popitem(last=False) for # eviction). Hard cap via _AGENT_CACHE_MAX_SIZE, idle TTL enforced # from _session_expiry_watcher(). import threading as _threading self._agent_cache: "OrderedDict[str, tuple]" = OrderedDict() self._agent_cache_lock = _threading.Lock() # Per-session model overrides from /model command. # Key: session_key, Value: dict with model/provider/api_key/base_url/api_mode self._session_model_overrides: Dict[str, Dict[str, str]] = {} # Per-session reasoning effort overrides from /reasoning. # Key: session_key, Value: parsed reasoning config dict. self._session_reasoning_overrides: Dict[str, Dict[str, Any]] = {} # Track pending exec approvals per session # Key: session_key, Value: {"command": str, "pattern_key": str, ...} self._pending_approvals: Dict[str, Dict[str, Any]] = {} # Track platforms that failed to connect for background reconnection. # Key: Platform enum, Value: {"config": platform_config, "attempts": int, "next_retry": float} self._failed_platforms: Dict[Platform, Dict[str, Any]] = {} # Track pending /update prompt responses per session. # Key: session_key, Value: True when a prompt is waiting for user input. self._update_prompt_pending: Dict[str, bool] = {} # Slash-confirm state lives in tools.slash_confirm (module-level), # so platform adapters can resolve callbacks without a backref to # this runner. Keep a local counter for confirm_id generation so # IDs stay compact (button callback_data has a 64-byte cap on # some platforms). import itertools as _itertools self._slash_confirm_counter = _itertools.count(1) # Persistent Honcho managers keyed by gateway session key. # This preserves write_frequency="session" semantics across short-lived # per-message AIAgent instances. # Ensure tirith security scanner is available (downloads if needed) try: from tools.tirith_security import ensure_installed ensure_installed(log_failures=False) except Exception: pass # Non-fatal — fail-open at scan time if unavailable # Initialize session database for session_search tool support self._session_db = None try: from hermes_state import SessionDB self._session_db = SessionDB() except Exception as e: logger.debug("SQLite session store not available: %s", e) # Opportunistic state.db maintenance: prune ended sessions older # than sessions.retention_days + optional VACUUM. Tracks last-run # in state_meta so it only actually executes once per # sessions.min_interval_hours. Gateway is long-lived so blocking # a few seconds once per day is acceptable; failures are logged # but never raised. if self._session_db is not None: try: from hermes_cli.config import load_config as _load_full_config _sess_cfg = (_load_full_config().get("sessions") or {}) if _sess_cfg.get("auto_prune", False): self._session_db.maybe_auto_prune_and_vacuum( retention_days=int(_sess_cfg.get("retention_days", 90)), min_interval_hours=int(_sess_cfg.get("min_interval_hours", 24)), vacuum=bool(_sess_cfg.get("vacuum_after_prune", True)), sessions_dir=self.config.sessions_dir, ) except Exception as exc: logger.debug("state.db auto-maintenance skipped: %s", exc) # Opportunistic shadow-repo cleanup — deletes orphan/stale # checkpoint repos under ~/.hermes/checkpoints/. Opt-in via # checkpoints.auto_prune, idempotent via .last_prune marker. try: from hermes_cli.config import load_config as _load_full_config _ckpt_cfg = (_load_full_config().get("checkpoints") or {}) if _ckpt_cfg.get("auto_prune", False): from tools.checkpoint_manager import maybe_auto_prune_checkpoints maybe_auto_prune_checkpoints( retention_days=int(_ckpt_cfg.get("retention_days", 7)), min_interval_hours=int(_ckpt_cfg.get("min_interval_hours", 24)), delete_orphans=bool(_ckpt_cfg.get("delete_orphans", True)), ) except Exception as exc: logger.debug("checkpoint auto-maintenance skipped: %s", exc) # DM pairing store for code-based user authorization from gateway.pairing import PairingStore self.pairing_store = PairingStore() # Event hook system from gateway.hooks import HookRegistry self.hooks = HookRegistry() # Per-chat voice reply mode: "off" | "voice_only" | "all" self._voice_mode: Dict[str, str] = self._load_voice_modes() # Track background tasks to prevent garbage collection mid-execution self._background_tasks: set = set() def _warn_if_docker_media_delivery_is_risky(self) -> None: """Warn when Docker-backed gateways lack an explicit export mount. MEDIA delivery happens in the gateway process, so paths emitted by the model must be readable from the host. A plain container-local path like `/workspace/report.txt` or `/output/report.txt` often exists only inside Docker, so users commonly need a dedicated export mount such as `host-dir:/output`. """ if os.getenv("TERMINAL_ENV", "").strip().lower() != "docker": return connected = self.config.get_connected_platforms() messaging_platforms = [p for p in connected if p not in {Platform.LOCAL, Platform.API_SERVER, Platform.WEBHOOK}] if not messaging_platforms: return raw_volumes = os.getenv("TERMINAL_DOCKER_VOLUMES", "").strip() volumes: List[str] = [] if raw_volumes: try: parsed = json.loads(raw_volumes) if isinstance(parsed, list): volumes = [str(v) for v in parsed if isinstance(v, str)] except Exception: logger.debug("Could not parse TERMINAL_DOCKER_VOLUMES for gateway media warning", exc_info=True) has_explicit_output_mount = False for spec in volumes: match = _DOCKER_VOLUME_SPEC_RE.match(spec) if not match: continue container_path = match.group("container") if container_path in _DOCKER_MEDIA_OUTPUT_CONTAINER_PATHS: has_explicit_output_mount = True break if has_explicit_output_mount: return logger.warning( "Docker backend is enabled for the messaging gateway but no explicit host-visible " "output mount (for example '/home/user/.hermes/cache/documents:/output') is configured. " "This is fine if the model already emits host-visible paths, but MEDIA file delivery can fail " "for container-local paths like '/workspace/...' or '/output/...'." ) # -- Setup skill availability ---------------------------------------- def _has_setup_skill(self) -> bool: """Check if the hermes-agent-setup skill is installed.""" try: from tools.skill_manager_tool import _find_skill return _find_skill("hermes-agent-setup") is not None except Exception: return False # -- Voice mode persistence ------------------------------------------ _VOICE_MODE_PATH = _hermes_home / "gateway_voice_mode.json" def _voice_key(self, platform: Platform, chat_id: str) -> str: """Return a platform-namespaced key for voice mode state.""" return f"{platform.value}:{chat_id}" def _load_voice_modes(self) -> Dict[str, str]: try: data = json.loads(self._VOICE_MODE_PATH.read_text()) except (FileNotFoundError, json.JSONDecodeError, OSError): return {} if not isinstance(data, dict): return {} valid_modes = {"off", "voice_only", "all"} result = {} for chat_id, mode in data.items(): if mode not in valid_modes: continue key = str(chat_id) # Skip legacy unprefixed keys (warn and skip) if ":" not in key: logger.warning( "Skipping legacy unprefixed voice mode key %r during migration. " "Re-enable voice mode on that chat to rebuild the prefixed key.", key, ) continue result[key] = mode return result def _save_voice_modes(self) -> None: try: self._VOICE_MODE_PATH.parent.mkdir(parents=True, exist_ok=True) self._VOICE_MODE_PATH.write_text( json.dumps(self._voice_mode, indent=2) ) except OSError as e: logger.warning("Failed to save voice modes: %s", e) def _set_adapter_auto_tts_disabled(self, adapter, chat_id: str, disabled: bool) -> None: """Update an adapter's in-memory auto-TTS suppression set if present.""" disabled_chats = getattr(adapter, "_auto_tts_disabled_chats", None) if not isinstance(disabled_chats, set): return if disabled: disabled_chats.add(chat_id) # ``/voice off`` also clears any explicit enable — it's a hard override. enabled_chats = getattr(adapter, "_auto_tts_enabled_chats", None) if isinstance(enabled_chats, set): enabled_chats.discard(chat_id) else: disabled_chats.discard(chat_id) def _set_adapter_auto_tts_enabled(self, adapter, chat_id: str, enabled: bool) -> None: """Update an adapter's per-chat auto-TTS opt-in set if present. Used for ``/voice on``/``/voice tts`` where the user explicitly wants auto-TTS even when ``voice.auto_tts`` is False globally. """ enabled_chats = getattr(adapter, "_auto_tts_enabled_chats", None) if not isinstance(enabled_chats, set): return if enabled: enabled_chats.add(chat_id) # An explicit opt-in clears any stale /voice off for this chat. disabled_chats = getattr(adapter, "_auto_tts_disabled_chats", None) if isinstance(disabled_chats, set): disabled_chats.discard(chat_id) else: enabled_chats.discard(chat_id) def _sync_voice_mode_state_to_adapter(self, adapter) -> None: """Restore persisted /voice state into a live platform adapter. Populates three fields from config + ``self._voice_mode``: - ``_auto_tts_default``: global default from ``voice.auto_tts`` - ``_auto_tts_enabled_chats``: chats with mode ``voice_only``/``all`` - ``_auto_tts_disabled_chats``: chats with mode ``off`` """ platform = getattr(adapter, "platform", None) if not isinstance(platform, Platform): return disabled_chats = getattr(adapter, "_auto_tts_disabled_chats", None) enabled_chats = getattr(adapter, "_auto_tts_enabled_chats", None) if not isinstance(disabled_chats, set) and not isinstance(enabled_chats, set): return # Push the global voice.auto_tts default (config.yaml) onto the adapter. # Lazy import to avoid adding a module-level dep from gateway → hermes_cli. try: from hermes_cli.config import load_config as _load_full_config _full_cfg = _load_full_config() _auto_tts_default = bool( (_full_cfg.get("voice") or {}).get("auto_tts", False) ) except Exception: _auto_tts_default = False if hasattr(adapter, "_auto_tts_default"): adapter._auto_tts_default = _auto_tts_default prefix = f"{platform.value}:" if isinstance(disabled_chats, set): disabled_chats.clear() disabled_chats.update( key[len(prefix):] for key, mode in self._voice_mode.items() if mode == "off" and key.startswith(prefix) ) if isinstance(enabled_chats, set): enabled_chats.clear() enabled_chats.update( key[len(prefix):] for key, mode in self._voice_mode.items() if mode in ("voice_only", "all") and key.startswith(prefix) ) async def _safe_adapter_disconnect(self, adapter, platform) -> None: """Call adapter.disconnect() defensively, swallowing any error. Used when adapter.connect() failed or raised — the adapter may have allocated partial resources (aiohttp.ClientSession, poll tasks, child subprocesses) that would otherwise leak and surface as "Unclosed client session" warnings at process exit. Must tolerate partial-init state and never raise, since callers use it inside error-handling blocks. """ try: await adapter.disconnect() except Exception as e: logger.debug( "Defensive %s disconnect after failed connect raised: %s", platform.value if platform is not None else "adapter", e, ) def _platform_connect_timeout_secs(self) -> float: """Return the per-platform connect timeout used during startup/retry.""" raw = os.getenv("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "").strip() if raw: try: timeout = float(raw) except ValueError: logger.warning( "Ignoring invalid HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT=%r", raw, ) else: return max(0.0, timeout) return _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT async def _connect_adapter_with_timeout(self, adapter, platform) -> bool: """Connect an adapter without allowing one platform to block others.""" timeout = self._platform_connect_timeout_secs() if timeout <= 0: return await adapter.connect() try: return await asyncio.wait_for(adapter.connect(), timeout=timeout) except asyncio.TimeoutError as exc: raise TimeoutError( f"{platform.value} connect timed out after {timeout:g}s" ) from exc @property def should_exit_cleanly(self) -> bool: return self._exit_cleanly @property def should_exit_with_failure(self) -> bool: return self._exit_with_failure @property def exit_reason(self) -> Optional[str]: return self._exit_reason @property def exit_code(self) -> Optional[int]: return self._exit_code def _session_key_for_source(self, source: SessionSource) -> str: """Resolve the current session key for a source, honoring gateway config when available.""" if hasattr(self, "session_store") and self.session_store is not None: try: session_key = self.session_store._generate_session_key(source) if isinstance(session_key, str) and session_key: return session_key except Exception: pass config = getattr(self, "config", None) return build_session_key( source, group_sessions_per_user=getattr(config, "group_sessions_per_user", True), thread_sessions_per_user=getattr(config, "thread_sessions_per_user", False), ) def _resolve_session_agent_runtime( self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, user_config: Optional[dict] = None, ) -> tuple[str, dict]: """Resolve model/runtime for a session, honoring session-scoped /model overrides. If the session override already contains a complete provider bundle (provider/api_key/base_url/api_mode), prefer it directly instead of resolving fresh global runtime state first. """ resolved_session_key = session_key if not resolved_session_key and source is not None: try: resolved_session_key = self._session_key_for_source(source) except Exception: resolved_session_key = None model = _resolve_gateway_model(user_config) override = self._session_model_overrides.get(resolved_session_key) if resolved_session_key else None if override: override_model = override.get("model", model) override_runtime = { "provider": override.get("provider"), "api_key": override.get("api_key"), "base_url": override.get("base_url"), "api_mode": override.get("api_mode"), } if override_runtime.get("api_key"): logger.debug( "Session model override (fast): session=%s config_model=%s -> override_model=%s provider=%s", resolved_session_key or "", model, override_model, override_runtime.get("provider"), ) return override_model, override_runtime # Override exists but has no api_key — fall through to env-based # resolution and apply model/provider from the override on top. logger.debug( "Session model override (no api_key, fallback): session=%s config_model=%s override_model=%s", resolved_session_key or "", model, override_model, ) else: logger.debug( "No session model override: session=%s config_model=%s override_keys=%s", resolved_session_key or "", model, list(self._session_model_overrides.keys())[:5] if self._session_model_overrides else "[]", ) runtime_kwargs = _resolve_runtime_agent_kwargs() if override and resolved_session_key: model, runtime_kwargs = self._apply_session_model_override( resolved_session_key, model, runtime_kwargs ) # When the config has no model.default but a provider was resolved # (e.g. user ran `hermes auth add openai-codex` without `hermes model`), # fall back to the provider's first catalog model so the API call # doesn't fail with "model must be a non-empty string". if not model and runtime_kwargs.get("provider"): try: from hermes_cli.models import get_default_model_for_provider model = get_default_model_for_provider(runtime_kwargs["provider"]) if model: logger.info( "No model configured — defaulting to %s for provider %s", model, runtime_kwargs["provider"], ) except Exception: pass return model, runtime_kwargs def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict: """Build the effective model/runtime config for a single turn. Always uses the session's primary model/provider. If `/fast` is enabled and the model supports Priority Processing / Anthropic fast mode, attach `request_overrides` so the API call is marked accordingly. """ from hermes_cli.models import resolve_fast_mode_overrides runtime = { "api_key": runtime_kwargs.get("api_key"), "base_url": runtime_kwargs.get("base_url"), "provider": runtime_kwargs.get("provider"), "api_mode": runtime_kwargs.get("api_mode"), "command": runtime_kwargs.get("command"), "args": list(runtime_kwargs.get("args") or []), "credential_pool": runtime_kwargs.get("credential_pool"), } route = { "model": model, "runtime": runtime, "signature": ( model, runtime["provider"], runtime["base_url"], runtime["api_mode"], runtime["command"], tuple(runtime["args"]), ), } service_tier = getattr(self, "_service_tier", None) if not service_tier: route["request_overrides"] = {} return route try: overrides = resolve_fast_mode_overrides(route["model"]) except Exception: overrides = None route["request_overrides"] = overrides or {} return route async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None: """React to an adapter failure after startup. If the error is retryable (e.g. network blip, DNS failure), queue the platform for background reconnection instead of giving up permanently. """ logger.error( "Fatal %s adapter error (%s): %s", adapter.platform.value, adapter.fatal_error_code or "unknown", adapter.fatal_error_message or "unknown error", ) self._update_platform_runtime_status( adapter.platform.value, platform_state="retrying" if adapter.fatal_error_retryable else "fatal", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) existing = self.adapters.get(adapter.platform) if existing is adapter: try: await adapter.disconnect() finally: self.adapters.pop(adapter.platform, None) self.delivery_router.adapters = self.adapters # Queue retryable failures for background reconnection if adapter.fatal_error_retryable: platform_config = self.config.platforms.get(adapter.platform) if platform_config and adapter.platform not in self._failed_platforms: self._failed_platforms[adapter.platform] = { "config": platform_config, "attempts": 0, "next_retry": time.monotonic() + 30, } logger.info( "%s queued for background reconnection", adapter.platform.value, ) if not self.adapters and not self._failed_platforms: self._exit_reason = adapter.fatal_error_message or "All messaging adapters disconnected" if adapter.fatal_error_retryable: self._exit_with_failure = True logger.error("No connected messaging platforms remain. Shutting down gateway for service restart.") else: logger.error("No connected messaging platforms remain. Shutting down gateway cleanly.") await self.stop() elif not self.adapters and self._failed_platforms: # All platforms are down and queued for background reconnection. # If the error is retryable, exit with failure so systemd Restart=on-failure # can restart the process. Otherwise stay alive and keep retrying in background. if adapter.fatal_error_retryable: self._exit_reason = adapter.fatal_error_message or "All messaging platforms failed with retryable errors" self._exit_with_failure = True logger.error( "All messaging platforms failed with retryable errors. " "Shutting down gateway for service restart (systemd will retry)." ) await self.stop() else: logger.warning( "No connected messaging platforms remain, but %d platform(s) queued for reconnection", len(self._failed_platforms), ) def _request_clean_exit(self, reason: str) -> None: self._exit_cleanly = True self._exit_reason = reason self._shutdown_event.set() def _running_agent_count(self) -> int: return len(self._running_agents) def _status_action_label(self) -> str: return "restart" if self._restart_requested else "shutdown" def _status_action_gerund(self) -> str: return "restarting" if self._restart_requested else "shutting down" def _queue_during_drain_enabled(self) -> bool: # Both "queue" and "steer" modes imply the user doesn't want messages # to be lost during restart — queue them for the newly-spawned gateway # process to pick up. "interrupt" mode drops them (current behaviour). return self._restart_requested and self._busy_input_mode in ("queue", "steer") # -------- /queue FIFO helpers -------------------------------------- # /queue must produce one full agent turn per invocation, in FIFO # order, with no merging. The adapter's _pending_messages dict is a # single "next-up" slot (shared with photo-burst follow-ups), so we # use it for the head of the queue and an overflow list for the # tail. Enqueue puts new items in the slot when free, otherwise in # the overflow. Promotion (called after each run's drain) moves the # next overflow item into the slot so the following recursion picks # it up. Clearing happens on /new and /reset via # _handle_reset_command. def _enqueue_fifo(self, session_key: str, queued_event: "MessageEvent", adapter: Any) -> None: """Append a /queue event to the FIFO chain for a session.""" if adapter is None: return pending_slot = getattr(adapter, "_pending_messages", None) if pending_slot is None: return queued_events = getattr(self, "_queued_events", None) if queued_events is None: queued_events = {} self._queued_events = queued_events if session_key in pending_slot: queued_events.setdefault(session_key, []).append(queued_event) else: pending_slot[session_key] = queued_event def _promote_queued_event( self, session_key: str, adapter: Any, pending_event: Optional["MessageEvent"], ) -> Optional["MessageEvent"]: """Promote the next overflow item after the slot was drained. Called at the drain site after _dequeue_pending_event consumed (or failed to consume) the slot. If there's an overflow item: - When pending_event is None (slot was empty), return the overflow head as the new pending_event. - When pending_event already exists (slot was populated by an interrupt follow-up or similar), stage the overflow head in the slot so the NEXT recursion picks it up. Returns the (possibly updated) pending_event for drain to use. """ queued_events = getattr(self, "_queued_events", None) if not queued_events: return pending_event overflow = queued_events.get(session_key) if not overflow: return pending_event next_queued = overflow.pop(0) if not overflow: queued_events.pop(session_key, None) if pending_event is None: return next_queued if adapter is not None and hasattr(adapter, "_pending_messages"): adapter._pending_messages[session_key] = next_queued else: # No adapter — push back so we don't silently drop the item. queued_events.setdefault(session_key, []).insert(0, next_queued) return pending_event def _queue_depth(self, session_key: str, *, adapter: Any = None) -> int: """Total pending /queue items for a session — slot + overflow.""" queued_events = getattr(self, "_queued_events", None) or {} depth = len(queued_events.get(session_key, [])) if adapter is not None and session_key in getattr(adapter, "_pending_messages", {}): depth += 1 return depth def _update_runtime_status(self, gateway_state: Optional[str] = None, exit_reason: Optional[str] = None) -> None: try: from gateway.status import write_runtime_status write_runtime_status( gateway_state=gateway_state, exit_reason=exit_reason, restart_requested=self._restart_requested, active_agents=self._running_agent_count(), ) except Exception: pass def _update_platform_runtime_status( self, platform: str, *, platform_state: Optional[str] = None, error_code: Optional[str] = None, error_message: Optional[str] = None, ) -> None: try: from gateway.status import write_runtime_status write_runtime_status( platform=platform, platform_state=platform_state, error_code=error_code, error_message=error_message, ) except Exception: pass @staticmethod def _load_prefill_messages() -> List[Dict[str, Any]]: """Load ephemeral prefill messages from config or env var. Checks HERMES_PREFILL_MESSAGES_FILE env var first, then falls back to the prefill_messages_file key in ~/.hermes/config.yaml. Relative paths are resolved from ~/.hermes/. """ file_path = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "") if not file_path: try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} file_path = cfg.get("prefill_messages_file", "") except Exception: pass if not file_path: return [] path = Path(file_path).expanduser() if not path.is_absolute(): path = _hermes_home / path if not path.exists(): logger.warning("Prefill messages file not found: %s", path) return [] try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) if not isinstance(data, list): logger.warning("Prefill messages file must contain a JSON array: %s", path) return [] return data except Exception as e: logger.warning("Failed to load prefill messages from %s: %s", path, e) return [] @staticmethod def _load_ephemeral_system_prompt() -> str: """Load ephemeral system prompt from config or env var. Checks HERMES_EPHEMERAL_SYSTEM_PROMPT env var first, then falls back to agent.system_prompt in ~/.hermes/config.yaml. """ prompt = os.getenv("HERMES_EPHEMERAL_SYSTEM_PROMPT", "") if prompt: return prompt try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} return (cfg_get(cfg, "agent", "system_prompt", default="") or "").strip() except Exception: pass return "" @staticmethod def _load_reasoning_config() -> dict | None: """Load reasoning effort from config.yaml. Reads agent.reasoning_effort from config.yaml. Valid: "none", "minimal", "low", "medium", "high", "xhigh". Returns None to use default (medium). """ from hermes_constants import parse_reasoning_effort effort = "" try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} effort = str(cfg_get(cfg, "agent", "reasoning_effort", default="") or "").strip() except Exception: pass result = parse_reasoning_effort(effort) if effort and effort.strip() and result is None: logger.warning("Unknown reasoning_effort '%s', using default (medium)", effort) return result @staticmethod def _parse_reasoning_command_args(raw_args: str) -> tuple[str, bool]: """Parse `/reasoning` args into `(value, persist_global)`. `/reasoning ` is session-scoped by default. `--global` may be supplied in any position to persist the change to config.yaml. """ import shlex text = str(raw_args or "").strip().replace("—", "--") if not text: return "", False try: tokens = shlex.split(text) except ValueError: tokens = text.split() persist_global = False value_tokens = [] for token in tokens: if token == "--global": persist_global = True else: value_tokens.append(token) return " ".join(value_tokens).strip().lower(), persist_global def _resolve_session_reasoning_config( self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, ) -> dict | None: """Resolve reasoning effort for a session, honoring session overrides.""" resolved_session_key = session_key if not resolved_session_key and source is not None: try: resolved_session_key = self._session_key_for_source(source) except Exception: resolved_session_key = None overrides = getattr(self, "_session_reasoning_overrides", {}) or {} if resolved_session_key and resolved_session_key in overrides: return overrides[resolved_session_key] return self._load_reasoning_config() def _set_session_reasoning_override( self, session_key: str, reasoning_config: Optional[dict], ) -> None: """Set or clear the session-scoped reasoning override.""" if not session_key: return if not hasattr(self, "_session_reasoning_overrides"): self._session_reasoning_overrides = {} if reasoning_config is None: self._session_reasoning_overrides.pop(session_key, None) else: self._session_reasoning_overrides[session_key] = dict(reasoning_config) @staticmethod def _load_service_tier() -> str | None: """Load Priority Processing setting from config.yaml. Reads agent.service_tier from config.yaml. Accepted values mirror the CLI: "fast"/"priority"/"on" => "priority", while "normal"/"off" disables it. Returns None when unset or unsupported. """ raw = "" try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} raw = str(cfg_get(cfg, "agent", "service_tier", default="") or "").strip() except Exception: pass value = raw.lower() if not value or value in {"normal", "default", "standard", "off", "none"}: return None if value in {"fast", "priority", "on"}: return "priority" logger.warning("Unknown service_tier '%s', ignoring", raw) return None @staticmethod def _load_show_reasoning() -> bool: """Load show_reasoning toggle from config.yaml display section.""" try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} return is_truthy_value( cfg_get(cfg, "display", "show_reasoning"), default=False, ) except Exception: pass return False @staticmethod def _load_busy_input_mode() -> str: """Load gateway drain-time busy-input behavior from config/env.""" mode = os.getenv("HERMES_GATEWAY_BUSY_INPUT_MODE", "").strip().lower() if not mode: try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} mode = str(cfg_get(cfg, "display", "busy_input_mode", default="") or "").strip().lower() except Exception: pass if mode == "queue": return "queue" if mode == "steer": return "steer" return "interrupt" @staticmethod def _load_restart_drain_timeout() -> float: """Load graceful gateway restart/stop drain timeout in seconds.""" raw = os.getenv("HERMES_RESTART_DRAIN_TIMEOUT", "").strip() if not raw: try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} raw = str(cfg_get(cfg, "agent", "restart_drain_timeout", default="") or "").strip() except Exception: pass value = parse_restart_drain_timeout(raw) if raw and value == DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT: try: float(raw) except (TypeError, ValueError): logger.warning( "Invalid restart_drain_timeout '%s', using default %.0fs", raw, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, ) return value @staticmethod def _load_background_notifications_mode() -> str: """Load background process notification mode from config or env var. Modes: - ``all`` — push running-output updates *and* the final message (default) - ``result`` — only the final completion message (regardless of exit code) - ``error`` — only the final message when exit code is non-zero - ``off`` — no watcher messages at all """ mode = os.getenv("HERMES_BACKGROUND_NOTIFICATIONS", "") if not mode: try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} raw = cfg_get(cfg, "display", "background_process_notifications") if raw is False: mode = "off" elif raw not in (None, ""): mode = str(raw) except Exception: pass mode = (mode or "all").strip().lower() valid = {"all", "result", "error", "off"} if mode not in valid: logger.warning( "Unknown background_process_notifications '%s', defaulting to 'all'", mode, ) return "all" return mode @staticmethod def _load_provider_routing() -> dict: """Load OpenRouter provider routing preferences from config.yaml.""" try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} return cfg.get("provider_routing", {}) or {} except Exception: pass return {} @staticmethod def _load_fallback_model() -> list | dict | None: """Load fallback provider chain from config.yaml. Returns a list of provider dicts (``fallback_providers``), a single dict (legacy ``fallback_model``), or None if not configured. AIAgent.__init__ normalizes both formats into a chain. """ try: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} fb = cfg.get("fallback_providers") or cfg.get("fallback_model") or None if fb: return fb except Exception: pass return None def _snapshot_running_agents(self) -> Dict[str, Any]: return { session_key: agent for session_key, agent in self._running_agents.items() if agent is not _AGENT_PENDING_SENTINEL } def _queue_or_replace_pending_event(self, session_key: str, event: MessageEvent) -> None: adapter = self.adapters.get(event.source.platform) if not adapter: return merge_pending_message_event(adapter._pending_messages, session_key, event) async def _handle_active_session_busy_message(self, event: MessageEvent, session_key: str) -> bool: # --- Authorization gate (#17775) --- # The cold path (_handle_message) checks _is_user_authorized before # creating a session. The busy path must enforce the same check; # otherwise unauthorized users in shared threads (Slack/Telegram/Discord) # can inject messages into an active session they don't own. if not self._is_user_authorized(event.source): logger.warning( "Dropping message from unauthorized user in active session: " "user=%s (%s), platform=%s, session=%s", event.source.user_id, event.source.user_name, event.source.platform.value if event.source.platform else "unknown", session_key, ) return True # handled (silently dropped); do not fall through # --- Draining case (gateway restarting/stopping) --- if self._draining: adapter = self.adapters.get(event.source.platform) if not adapter: return True thread_meta = {"thread_id": event.source.thread_id} if event.source.thread_id else None if self._queue_during_drain_enabled(): self._queue_or_replace_pending_event(session_key, event) message = f"⏳ Gateway {self._status_action_gerund()} — queued for the next turn after it comes back." else: message = f"⏳ Gateway is {self._status_action_gerund()} and is not accepting another turn right now." await adapter._send_with_retry( chat_id=event.source.chat_id, content=message, reply_to=event.message_id, metadata=thread_meta, ) return True # Normal busy case (agent actively running a task) adapter = self.adapters.get(event.source.platform) if not adapter: return False # let default path handle it running_agent = self._running_agents.get(session_key) # Steer mode: inject mid-run via running_agent.steer() instead of # queueing + interrupting. If the agent isn't running yet # (sentinel) or lacks steer(), or the payload is empty, fall back # to queue semantics so nothing is lost. effective_mode = self._busy_input_mode steered = False if effective_mode == "steer": steer_text = (event.text or "").strip() can_steer = ( steer_text and running_agent is not None and running_agent is not _AGENT_PENDING_SENTINEL and hasattr(running_agent, "steer") ) if can_steer: try: steered = bool(running_agent.steer(steer_text)) except Exception as exc: logger.warning("Gateway steer failed for session %s: %s", session_key, exc) steered = False if not steered: # Fall back to queue (merge into pending messages, no interrupt) effective_mode = "queue" # Store the message so it's processed as the next turn after the # current run finishes (or is interrupted). Skip this for a # successful steer — the text already landed inside the run and # must NOT also be replayed as a next-turn user message. if not steered: merge_pending_message_event(adapter._pending_messages, session_key, event) is_queue_mode = effective_mode == "queue" is_steer_mode = effective_mode == "steer" # If not in queue/steer mode, interrupt the running agent immediately. # This aborts in-flight tool calls and causes the agent loop to exit # at the next check point. if effective_mode == "interrupt" and running_agent and running_agent is not _AGENT_PENDING_SENTINEL: try: running_agent.interrupt(event.text) except Exception: pass # don't let interrupt failure block the ack # Check if busy ack is disabled — skip sending but still process the input. # Placed before debounce so we don't stamp a "last ack" timestamp that was # never actually delivered. busy_ack_enabled = os.environ.get("HERMES_GATEWAY_BUSY_ACK_ENABLED", "true").lower() == "true" if not busy_ack_enabled: logger.debug("Busy ack suppressed for session %s", session_key) return True # input still processed, just no ack sent # Debounce: only send an acknowledgment once every 30 seconds per session # to avoid spamming the user when they send multiple messages quickly _BUSY_ACK_COOLDOWN = 30 now = time.time() last_ack = self._busy_ack_ts.get(session_key, 0) if now - last_ack < _BUSY_ACK_COOLDOWN: return True # interrupt sent (if not queue), ack already delivered recently self._busy_ack_ts[session_key] = now # Build a status-rich acknowledgment status_parts = [] if running_agent and running_agent is not _AGENT_PENDING_SENTINEL: try: summary = running_agent.get_activity_summary() iteration = summary.get("api_call_count", 0) max_iter = summary.get("max_iterations", 0) current_tool = summary.get("current_tool") start_ts = self._running_agents_ts.get(session_key, 0) if start_ts: elapsed_min = int((now - start_ts) / 60) if elapsed_min > 0: status_parts.append(f"{elapsed_min} min elapsed") if max_iter: status_parts.append(f"iteration {iteration}/{max_iter}") if current_tool: status_parts.append(f"running: {current_tool}") except Exception: pass status_detail = f" ({', '.join(status_parts)})" if status_parts else "" if is_steer_mode: message = ( f"⏩ Steered into current run{status_detail}. " f"Your message arrives after the next tool call." ) elif is_queue_mode: message = ( f"⏳ Queued for the next turn{status_detail}. " f"I'll respond once the current task finishes." ) else: message = ( f"⚡ Interrupting current task{status_detail}. " f"I'll respond to your message shortly." ) # First-touch onboarding: the very first time a user sends a message # while the agent is busy, append a one-time hint explaining the # queue/interrupt knob. Flag is persisted to config.yaml so it never # fires again on this install. try: from agent.onboarding import ( BUSY_INPUT_FLAG, busy_input_hint_gateway, is_seen, mark_seen, ) _user_cfg = _load_gateway_config() if not is_seen(_user_cfg, BUSY_INPUT_FLAG): if is_steer_mode: _hint_mode = "steer" elif is_queue_mode: _hint_mode = "queue" else: _hint_mode = "interrupt" message = ( f"{message}\n\n" f"{busy_input_hint_gateway(_hint_mode)}" ) mark_seen(_hermes_home / "config.yaml", BUSY_INPUT_FLAG) except Exception as _onb_err: logger.debug("Failed to apply busy-input onboarding hint: %s", _onb_err) thread_meta = {"thread_id": event.source.thread_id} if event.source.thread_id else None try: await adapter._send_with_retry( chat_id=event.source.chat_id, content=message, reply_to=event.message_id, metadata=thread_meta, ) except Exception as e: logger.debug("Failed to send busy-ack: %s", e) return True async def _drain_active_agents(self, timeout: float) -> tuple[Dict[str, Any], bool]: snapshot = self._snapshot_running_agents() last_active_count = self._running_agent_count() last_status_at = 0.0 def _maybe_update_status(force: bool = False) -> None: nonlocal last_active_count, last_status_at now = asyncio.get_running_loop().time() active_count = self._running_agent_count() if force or active_count != last_active_count or (now - last_status_at) >= 1.0: self._update_runtime_status("draining") last_active_count = active_count last_status_at = now if not self._running_agents: _maybe_update_status(force=True) return snapshot, False _maybe_update_status(force=True) if timeout <= 0: return snapshot, True deadline = asyncio.get_running_loop().time() + timeout while self._running_agents and asyncio.get_running_loop().time() < deadline: _maybe_update_status() await asyncio.sleep(0.1) timed_out = bool(self._running_agents) _maybe_update_status(force=True) return snapshot, timed_out def _interrupt_running_agents(self, reason: str) -> None: for session_key, agent in list(self._running_agents.items()): if agent is _AGENT_PENDING_SENTINEL: continue try: agent.interrupt(reason) logger.debug("Interrupted running agent for session %s during shutdown", session_key) except Exception as e: logger.debug("Failed interrupting agent during shutdown: %s", e) async def _notify_active_sessions_of_shutdown(self) -> None: """Send a notification to every chat with an active agent. Called at the very start of stop() — adapters are still connected so messages can be delivered. Best-effort: individual send failures are logged and swallowed so they never block the shutdown sequence. """ active = self._snapshot_running_agents() if not active: return action = "restarting" if self._restart_requested else "shutting down" hint = ( "Your current task will be interrupted. " "Send any message after restart and I'll try to resume where you left off." if self._restart_requested else "Your current task will be interrupted." ) msg = f"⚠️ Gateway {action} — {hint}" notified: set = set() for session_key in active: source = None try: if getattr(self, "session_store", None) is not None: self.session_store._ensure_loaded() entry = self.session_store._entries.get(session_key) source = getattr(entry, "origin", None) if entry else None except Exception as e: logger.debug( "Failed to load session origin for shutdown notification %s: %s", session_key, e, ) if source is not None: platform_str = source.platform.value chat_id = source.chat_id thread_id = source.thread_id else: # Fall back to parsing the session key when no persisted # origin is available (legacy sessions/tests). _parsed = _parse_session_key(session_key) if not _parsed: continue platform_str = _parsed["platform"] chat_id = _parsed["chat_id"] thread_id = _parsed.get("thread_id") # Deduplicate: one notification per chat, even if multiple # sessions (different users/threads) share the same chat. dedup_key = (platform_str, chat_id) if dedup_key in notified: continue try: platform = Platform(platform_str) adapter = self.adapters.get(platform) if not adapter: continue # Include thread_id if present so the message lands in the # correct forum topic / thread. metadata = {"thread_id": thread_id} if thread_id else None await adapter.send(chat_id, msg, metadata=metadata) notified.add(dedup_key) logger.info( "Sent shutdown notification to %s:%s", platform_str, chat_id, ) except Exception as e: logger.debug( "Failed to send shutdown notification to %s:%s: %s", platform_str, chat_id, e, ) def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None: for agent in active_agents.values(): try: from hermes_cli.plugins import invoke_hook as _invoke_hook _invoke_hook( "on_session_finalize", session_id=getattr(agent, "session_id", None), platform="gateway", ) except Exception: pass self._cleanup_agent_resources(agent) def _cleanup_agent_resources(self, agent: Any) -> None: """Best-effort cleanup for temporary or cached agent instances.""" if agent is None: return try: if hasattr(agent, "shutdown_memory_provider"): # Pass the agent's own conversation transcript so memory # providers' ``on_session_end`` hooks see the real messages # instead of the empty default (#15165). ``_session_messages`` # is set on ``AIAgent`` (run_agent.py:1518) and refreshed at # the end of every ``run_conversation`` turn via # ``_persist_session``; on an agent built through # ``object.__new__`` (test stubs) the attribute may be # absent, so ``getattr`` with a ``None`` default keeps the # call signature-compatible with the pre-fix behaviour # (``shutdown_memory_provider(messages=None)``). session_messages = getattr(agent, "_session_messages", None) if isinstance(session_messages, list): agent.shutdown_memory_provider(session_messages) else: agent.shutdown_memory_provider() except Exception: pass # Close tool resources (terminal sandboxes, browser daemons, # background processes, httpx clients) to prevent zombie # process accumulation. try: if hasattr(agent, "close"): agent.close() except Exception: pass # Auxiliary async clients (session_search/web/vision/etc.) live in a # process-global cache and are created inside worker threads. Clean up # any entries whose event loop is now dead so their httpx transports do # not accumulate across gateway turns. try: from agent.auxiliary_client import cleanup_stale_async_clients cleanup_stale_async_clients() except Exception: pass _STUCK_LOOP_THRESHOLD = 3 # restarts while active before auto-suspend _STUCK_LOOP_FILE = ".restart_failure_counts" def _increment_restart_failure_counts(self, active_session_keys: set) -> None: """Increment restart-failure counters for sessions active at shutdown. Persists to a JSON file so counters survive across restarts. Sessions NOT in active_session_keys are removed (they completed successfully, so the loop is broken). """ import json path = _hermes_home / self._STUCK_LOOP_FILE try: counts = json.loads(path.read_text()) if path.exists() else {} except Exception: counts = {} # Increment active sessions, remove inactive ones (loop broken) new_counts = {} for key in active_session_keys: new_counts[key] = counts.get(key, 0) + 1 # Keep any entries that are still above 0 even if not active now # (they might become active again next restart) try: atomic_json_write(path, new_counts, indent=None) except Exception: pass def _suspend_stuck_loop_sessions(self) -> int: """Suspend sessions that have been active across too many restarts. Returns the number of sessions suspended. Called on gateway startup AFTER suspend_recently_active() to catch the stuck-loop pattern: session loads → agent gets stuck → gateway restarts → repeat. """ import json path = _hermes_home / self._STUCK_LOOP_FILE if not path.exists(): return 0 try: counts = json.loads(path.read_text()) except Exception: return 0 suspended = 0 stuck_keys = [k for k, v in counts.items() if v >= self._STUCK_LOOP_THRESHOLD] for session_key in stuck_keys: try: entry = self.session_store._entries.get(session_key) if entry and not entry.suspended: entry.suspended = True suspended += 1 logger.warning( "Auto-suspended stuck session %s (active across %d " "consecutive restarts — likely a stuck loop)", session_key, counts[session_key], ) except Exception: pass if suspended: try: self.session_store._save() except Exception: pass # Clear the file — counters start fresh after suspension try: path.unlink(missing_ok=True) except Exception: pass return suspended def _clear_restart_failure_count(self, session_key: str) -> None: """Clear the restart-failure counter for a session that completed OK. Called after a successful agent turn to signal the loop is broken. """ import json path = _hermes_home / self._STUCK_LOOP_FILE if not path.exists(): return try: counts = json.loads(path.read_text()) if session_key in counts: del counts[session_key] if counts: atomic_json_write(path, counts, indent=None) else: path.unlink(missing_ok=True) except Exception: pass async def _launch_detached_restart_command(self) -> None: import shutil import subprocess hermes_cmd = _resolve_hermes_bin() if not hermes_cmd: logger.error("Could not locate hermes binary for detached /restart") return current_pid = os.getpid() cmd = " ".join(shlex.quote(part) for part in hermes_cmd) shell_cmd = ( f"while kill -0 {current_pid} 2>/dev/null; do sleep 0.2; done; " f"{cmd} gateway restart" ) setsid_bin = shutil.which("setsid") if setsid_bin: subprocess.Popen( [setsid_bin, "bash", "-lc", shell_cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) else: subprocess.Popen( ["bash", "-lc", shell_cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) def request_restart(self, *, detached: bool = False, via_service: bool = False) -> bool: if self._restart_task_started: return False self._restart_requested = True self._restart_detached = detached self._restart_via_service = via_service self._restart_task_started = True async def _run_restart() -> None: await asyncio.sleep(0.05) await self.stop(restart=True, detached_restart=detached, service_restart=via_service) task = asyncio.create_task(_run_restart()) self._background_tasks.add(task) task.add_done_callback(self._background_tasks.discard) return True def _detect_stale_code(self) -> bool: """Return True if source files on disk are newer than the running process. A gateway that survives ``hermes update`` (manual SIGTERM never escalated, systemd restart race, detached-process respawn failed, etc.) keeps pre-update modules cached in ``sys.modules``. Later imports of names added post-update — e.g. ``cfg_get`` from PR #17304 — raise ImportError against the stale module object (see Issue #17648). Detecting this at the source — "the code on disk is newer than me" — lets us auto-restart instead of serving broken responses until the user notices and runs ``hermes gateway restart`` manually. Returns False when the boot-time snapshot is unavailable or no sentinel file is readable, to avoid false-positive restart loops in unusual checkouts (sparse clones, read-only filesystems). """ if not self._boot_wall_time or not self._boot_repo_mtime: return False try: current = _compute_repo_mtime(self._repo_root_for_staleness) except Exception: return False if current <= 0.0: return False # 2-second slack guards against filesystems with coarse mtime # resolution (FAT32, some NFS mounts). Real updates always move # the newest-file mtime forward by minutes, so this doesn't hide # genuine staleness. return current > self._boot_repo_mtime + 2.0 def _trigger_stale_code_restart(self) -> None: """Idempotently kick off a graceful restart after stale-code detection. Runs at most once per process. The restart request goes through the normal drain path so in-flight agent turns finish before the process exits; the service manager (systemd / launchd / detached profile watcher) then respawns with fresh code. On manual ``hermes gateway run`` installs without a supervisor, the process exits and the user must restart by hand — but they get a user-visible message telling them so. """ if self._stale_code_restart_triggered: return self._stale_code_restart_triggered = True logger.warning( "Stale-code self-check: source files newer than gateway boot " "time (boot=%.0f, newest=%.0f) — requesting graceful restart. " "See Issue #17648.", self._boot_repo_mtime, _compute_repo_mtime(self._repo_root_for_staleness), ) try: self.request_restart(detached=False, via_service=True) except Exception as exc: logger.error("Stale-code restart request failed: %s", exc) async def start(self) -> bool: """ Start the gateway and all configured platform adapters. Returns True if at least one adapter connected successfully. """ logger.info("Starting Hermes Gateway...") logger.info("Session storage: %s", self.config.sessions_dir) # Log the resolved max_iterations budget so operators can verify the # config.yaml → env bridge did the right thing at a glance (instead # of silently running at a stale .env value for weeks). try: _effective_max_iter = int(os.getenv("HERMES_MAX_ITERATIONS", "90")) logger.info( "Agent budget: max_iterations=%d (agent.max_turns from config.yaml, " "or HERMES_MAX_ITERATIONS from .env, or default 90)", _effective_max_iter, ) except Exception: pass try: from hermes_cli.profiles import get_active_profile_name _profile = get_active_profile_name() if _profile and _profile != "default": logger.info("Active profile: %s", _profile) except Exception: pass try: from gateway.status import write_runtime_status write_runtime_status(gateway_state="starting", exit_reason=None) except Exception: pass # Warn if no user allowlists are configured and open access is not opted in _builtin_allowed_vars = ( "TELEGRAM_ALLOWED_USERS", "DISCORD_ALLOWED_USERS", "WHATSAPP_ALLOWED_USERS", "SLACK_ALLOWED_USERS", "SIGNAL_ALLOWED_USERS", "SIGNAL_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_CHATS", "EMAIL_ALLOWED_USERS", "SMS_ALLOWED_USERS", "MATTERMOST_ALLOWED_USERS", "MATRIX_ALLOWED_USERS", "DINGTALK_ALLOWED_USERS", "FEISHU_ALLOWED_USERS", "WECOM_ALLOWED_USERS", "WECOM_CALLBACK_ALLOWED_USERS", "WEIXIN_ALLOWED_USERS", "BLUEBUBBLES_ALLOWED_USERS", "QQ_ALLOWED_USERS", "YUANBAO_ALLOWED_USERS", "GATEWAY_ALLOWED_USERS", ) _builtin_allow_all_vars = ( "TELEGRAM_ALLOW_ALL_USERS", "DISCORD_ALLOW_ALL_USERS", "WHATSAPP_ALLOW_ALL_USERS", "SLACK_ALLOW_ALL_USERS", "SIGNAL_ALLOW_ALL_USERS", "EMAIL_ALLOW_ALL_USERS", "SMS_ALLOW_ALL_USERS", "MATTERMOST_ALLOW_ALL_USERS", "MATRIX_ALLOW_ALL_USERS", "DINGTALK_ALLOW_ALL_USERS", "FEISHU_ALLOW_ALL_USERS", "WECOM_ALLOW_ALL_USERS", "WECOM_CALLBACK_ALLOW_ALL_USERS", "WEIXIN_ALLOW_ALL_USERS", "BLUEBUBBLES_ALLOW_ALL_USERS", "QQ_ALLOW_ALL_USERS", "YUANBAO_ALLOW_ALL_USERS", ) # Also pick up plugin-registered platforms — each entry can declare # its own allowed_users_env / allow_all_env, so the warning stays # accurate as plugins like IRC come online. _plugin_allowed_vars: tuple = () _plugin_allow_all_vars: tuple = () try: from gateway.platform_registry import platform_registry _plugin_allowed_vars = tuple( e.allowed_users_env for e in platform_registry.plugin_entries() if e.allowed_users_env ) _plugin_allow_all_vars = tuple( e.allow_all_env for e in platform_registry.plugin_entries() if e.allow_all_env ) except Exception: pass _any_allowlist = any( os.getenv(v) for v in _builtin_allowed_vars + _plugin_allowed_vars ) _allow_all = os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in ("true", "1", "yes") or any( os.getenv(v, "").lower() in ("true", "1", "yes") for v in _builtin_allow_all_vars + _plugin_allow_all_vars ) if not _any_allowlist and not _allow_all: logger.warning( "No user allowlists configured. All unauthorized users will be denied. " "Set GATEWAY_ALLOW_ALL_USERS=true in ~/.hermes/.env to allow open access, " "or configure platform allowlists (e.g., TELEGRAM_ALLOWED_USERS=your_id)." ) # Discover Python plugins before shell hooks so plugin block # decisions take precedence in tie cases. The CLI startup path # does this via an explicit call in hermes_cli/main.py; the # gateway lazily imports run_agent inside per-request handlers, # so the discover_plugins() side-effect in model_tools.py is NOT # guaranteed to have run by the time we reach this point. try: from hermes_cli.plugins import discover_plugins discover_plugins() except Exception: logger.debug( "plugin discovery failed at gateway startup", exc_info=True, ) # Register declarative shell hooks from cli-config.yaml. Gateway # has no TTY, so consent has to come from one of the three opt-in # channels (--accept-hooks on launch, HERMES_ACCEPT_HOOKS env var, # or hooks_auto_accept: true in config.yaml). We pass # accept_hooks=False here and let register_from_config resolve # the effective value from env + config itself — the CLI-side # registration already honored --accept-hooks, and re-reading # hooks_auto_accept here would just duplicate that lookup. # Failures are logged but must never block gateway startup. try: from hermes_cli.config import load_config from agent.shell_hooks import register_from_config register_from_config(load_config(), accept_hooks=False) except Exception: logger.debug( "shell-hook registration failed at gateway startup", exc_info=True, ) # Discover and load event hooks self.hooks.discover_and_load() # Recover background processes from checkpoint (crash recovery) try: from tools.process_registry import process_registry recovered = process_registry.recover_from_checkpoint() if recovered: logger.info("Recovered %s background process(es) from previous run", recovered) except Exception as e: logger.warning("Process checkpoint recovery: %s", e) # Suspend sessions that were active when the gateway last exited. # This prevents stuck sessions from being blindly resumed on restart, # which can create an unrecoverable loop (#7536). Suspended sessions # auto-reset on the next incoming message, giving the user a clean start. # # SKIP suspension after a clean (graceful) shutdown — the previous # process already drained active agents, so sessions aren't stuck. # This prevents unwanted auto-resets after `hermes update`, # `hermes gateway restart`, or `/restart`. _clean_marker = _hermes_home / ".clean_shutdown" if _clean_marker.exists(): logger.info("Previous gateway exited cleanly — skipping session suspension") try: _clean_marker.unlink() except Exception: pass else: try: suspended = self.session_store.suspend_recently_active() if suspended: logger.info("Marked %d in-flight session(s) as resumable from previous run", suspended) except Exception as e: logger.warning("Session suspension on startup failed: %s", e) # Stuck-loop detection (#7536): if a session has been active across # 3+ consecutive restarts, it's probably stuck in a loop (the same # history keeps causing the agent to hang). Auto-suspend it so the # user gets a clean slate on the next message. try: stuck = self._suspend_stuck_loop_sessions() if stuck: logger.warning("Auto-suspended %d stuck-loop session(s)", stuck) except Exception as e: logger.debug("Stuck-loop detection failed: %s", e) connected_count = 0 enabled_platform_count = 0 startup_nonretryable_errors: list[str] = [] startup_retryable_errors: list[str] = [] # Initialize and connect each configured platform for platform, platform_config in self.config.platforms.items(): if not platform_config.enabled: continue enabled_platform_count += 1 adapter = self._create_adapter(platform, platform_config) if not adapter: # Distinguish between missing builtin deps and missing plugin _pval = platform.value _builtin_names = {m.value for m in Platform.__members__.values()} if _pval not in _builtin_names: logger.warning( "No adapter for '%s' — is the plugin installed? " "(platform is enabled in config.yaml but no plugin registered it)", _pval, ) else: logger.warning("No adapter available for %s", _pval) continue # Set up message + fatal error handlers adapter.set_message_handler(self._handle_message) adapter.set_fatal_error_handler(self._handle_adapter_fatal_error) adapter.set_session_store(self.session_store) adapter.set_busy_session_handler(self._handle_active_session_busy_message) # Try to connect logger.info("Connecting to %s...", platform.value) self._update_platform_runtime_status( platform.value, platform_state="connecting", error_code=None, error_message=None, ) try: success = await self._connect_adapter_with_timeout(adapter, platform) if success: self.adapters[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) connected_count += 1 self._update_platform_runtime_status( platform.value, platform_state="connected", error_code=None, error_message=None, ) logger.info("✓ %s connected", platform.value) else: logger.warning("✗ %s failed to connect", platform.value) # Defensive cleanup: a failed connect() may have # allocated resources (aiohttp.ClientSession, poll # tasks, bridge subprocesses) before giving up. # Without this call, those resources are orphaned # and Python logs "Unclosed client session" at # process exit. Adapter disconnect() implementations # are expected to be idempotent and tolerate # partial-init state. await self._safe_adapter_disconnect(adapter, platform) if adapter.has_fatal_error: self._update_platform_runtime_status( platform.value, platform_state="retrying" if adapter.fatal_error_retryable else "fatal", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) target = ( startup_retryable_errors if adapter.fatal_error_retryable else startup_nonretryable_errors ) target.append( f"{platform.value}: {adapter.fatal_error_message}" ) # Queue for reconnection if the error is retryable if adapter.fatal_error_retryable: self._failed_platforms[platform] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, } else: self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=None, error_message="failed to connect", ) startup_retryable_errors.append( f"{platform.value}: failed to connect" ) # No fatal error info means likely a transient issue — queue for retry self._failed_platforms[platform] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, } except Exception as e: logger.error("✗ %s error: %s", platform.value, e) # Same defensive cleanup path for exceptions — an adapter # that raised mid-connect may still have a live # aiohttp.ClientSession or child subprocess. await self._safe_adapter_disconnect(adapter, platform) self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=None, error_message=str(e), ) startup_retryable_errors.append(f"{platform.value}: {e}") # Unexpected exceptions are typically transient — queue for retry self._failed_platforms[platform] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, } if connected_count == 0: if startup_nonretryable_errors: reason = "; ".join(startup_nonretryable_errors) logger.error("Gateway hit a non-retryable startup conflict: %s", reason) try: from gateway.status import write_runtime_status write_runtime_status(gateway_state="startup_failed", exit_reason=reason) except Exception: pass self._request_clean_exit(reason) return True if enabled_platform_count > 0: reason = "; ".join(startup_retryable_errors) or "all configured messaging platforms failed to connect" logger.error("Gateway failed to connect any configured messaging platform: %s", reason) try: from gateway.status import write_runtime_status write_runtime_status(gateway_state="startup_failed", exit_reason=reason) except Exception: pass return False logger.warning("No messaging platforms enabled.") logger.info("Gateway will continue running for cron job execution.") # Update delivery router with adapters self.delivery_router.adapters = self.adapters self._running = True self._update_runtime_status("running") # Emit gateway:startup hook hook_count = len(self.hooks.loaded_hooks) if hook_count: logger.info("%s hook(s) loaded", hook_count) await self.hooks.emit("gateway:startup", { "platforms": [p.value for p in self.adapters.keys()], }) if connected_count > 0: logger.info("Gateway running with %s platform(s)", connected_count) # Build initial channel directory for send_message name resolution try: from gateway.channel_directory import build_channel_directory directory = await build_channel_directory(self.adapters) ch_count = sum(len(chs) for chs in directory.get("platforms", {}).values()) logger.info("Channel directory built: %d target(s)", ch_count) except Exception as e: logger.warning("Channel directory build failed: %s", e) # Check if we're restarting after a /update command. If the update is # still running, keep watching so we notify once it actually finishes. notified = await self._send_update_notification() if not notified and any( path.exists() for path in ( _hermes_home / ".update_pending.json", _hermes_home / ".update_pending.claimed.json", ) ): self._schedule_update_notification_watch() # Notify the chat that initiated /restart that the gateway is back. await self._send_restart_notification() # Drain any recovered process watchers (from crash recovery checkpoint) try: from tools.process_registry import process_registry while process_registry.pending_watchers: watcher = process_registry.pending_watchers.pop(0) asyncio.create_task(self._run_process_watcher(watcher)) logger.info("Resumed watcher for recovered process %s", watcher.get("session_id")) except Exception as e: logger.error("Recovered watcher setup error: %s", e) # Start background session expiry watcher to finalize expired sessions asyncio.create_task(self._session_expiry_watcher()) # Start background kanban notifier — delivers `completed`, `blocked`, # `spawn_auto_blocked`, and `crashed` events to gateway subscribers # so human-in-the-loop workflows hear back without polling. asyncio.create_task(self._kanban_notifier_watcher()) # Start background kanban dispatcher — spawns workers for ready # tasks. Gated by `kanban.dispatch_in_gateway` (default True). # When false, users run `hermes kanban daemon` externally or # simply don't use kanban; this loop becomes a no-op. asyncio.create_task(self._kanban_dispatcher_watcher()) # Start background reconnection watcher for platforms that failed at startup if self._failed_platforms: logger.info( "Starting reconnection watcher for %d failed platform(s): %s", len(self._failed_platforms), ", ".join(p.value for p in self._failed_platforms), ) asyncio.create_task(self._platform_reconnect_watcher()) logger.info("Press Ctrl+C to stop") return True async def _session_expiry_watcher(self, interval: int = 300): """Background task that finalizes expired sessions. Runs every ``interval`` seconds (default 5 min). For each session whose reset policy has expired, invokes ``on_session_finalize`` hooks, cleans up the cached AIAgent's tool resources, evicts the cache entry so it can be garbage-collected, and marks the session so it won't be finalized again. """ await asyncio.sleep(60) # initial delay — let the gateway fully start _finalize_failures: dict[str, int] = {} # session_id -> consecutive failure count _MAX_FINALIZE_RETRIES = 3 while self._running: try: self.session_store._ensure_loaded() # Collect expired sessions first, then log a single summary. _expired_entries = [] for key, entry in list(self.session_store._entries.items()): if entry.expiry_finalized: continue if not self.session_store._is_session_expired(entry): continue _expired_entries.append((key, entry)) if _expired_entries: # Extract platform names from session keys for a compact summary. # Keys look like "agent:main:telegram:dm:12345" — platform is field [2]. _platforms: dict[str, int] = {} for _k, _e in _expired_entries: _parts = _k.split(":") _plat = _parts[2] if len(_parts) > 2 else "unknown" _platforms[_plat] = _platforms.get(_plat, 0) + 1 _plat_summary = ", ".join( f"{p}:{c}" for p, c in sorted(_platforms.items()) ) logger.info( "Session expiry: %d sessions to finalize (%s)", len(_expired_entries), _plat_summary, ) for key, entry in _expired_entries: try: try: from hermes_cli.plugins import invoke_hook as _invoke_hook _parts = key.split(":") _platform = _parts[2] if len(_parts) > 2 else "" _invoke_hook( "on_session_finalize", session_id=entry.session_id, platform=_platform, ) except Exception: pass # Shut down memory provider and close tool resources # on the cached agent. Idle agents live in # _agent_cache (not _running_agents), so look there. _cached_agent = None _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is not None: with _cache_lock: _cached = self._agent_cache.get(key) _cached_agent = _cached[0] if isinstance(_cached, tuple) else _cached if _cached else None # Fall back to _running_agents in case the agent is # still mid-turn when the expiry fires. if _cached_agent is None: _cached_agent = self._running_agents.get(key) if _cached_agent and _cached_agent is not _AGENT_PENDING_SENTINEL: self._cleanup_agent_resources(_cached_agent) # Drop the cache entry so the AIAgent (and its LLM # clients, tool schemas, memory provider refs) can # be garbage-collected. Otherwise the cache grows # unbounded across the gateway's lifetime. self._evict_cached_agent(key) # Mark as finalized and persist to disk so the flag # survives gateway restarts. with self.session_store._lock: entry.expiry_finalized = True self.session_store._save() logger.debug( "Session expiry finalized for %s", entry.session_id, ) _finalize_failures.pop(entry.session_id, None) except Exception as e: failures = _finalize_failures.get(entry.session_id, 0) + 1 _finalize_failures[entry.session_id] = failures if failures >= _MAX_FINALIZE_RETRIES: logger.warning( "Session finalize gave up after %d attempts for %s: %s. " "Marking as finalized to prevent infinite retry loop.", failures, entry.session_id, e, ) with self.session_store._lock: entry.expiry_finalized = True self.session_store._save() _finalize_failures.pop(entry.session_id, None) else: logger.debug( "Session finalize failed (%d/%d) for %s: %s", failures, _MAX_FINALIZE_RETRIES, entry.session_id, e, ) if _expired_entries: _done = sum( 1 for _, e in _expired_entries if e.expiry_finalized ) _failed = len(_expired_entries) - _done if _failed: logger.info( "Session expiry done: %d finalized, %d pending retry", _done, _failed, ) else: logger.info( "Session expiry done: %d finalized", _done, ) # Sweep agents that have been idle beyond the TTL regardless # of session reset policy. This catches sessions with very # long / "never" reset windows, whose cached AIAgents would # otherwise pin memory for the gateway's entire lifetime. try: _idle_evicted = self._sweep_idle_cached_agents() if _idle_evicted: logger.info( "Agent cache idle sweep: evicted %d agent(s)", _idle_evicted, ) except Exception as _e: logger.debug("Idle agent sweep failed: %s", _e) # Periodically prune stale SessionStore entries. The # in-memory dict (and sessions.json) would otherwise grow # unbounded in gateways serving many rotating chats / # threads / users over long time windows. Pruning is # invisible to users — a resumed session just gets a # fresh session_id, exactly as if the reset policy fired. _last_prune_ts = getattr(self, "_last_session_store_prune_ts", 0.0) _prune_interval = 3600.0 # once per hour if time.time() - _last_prune_ts > _prune_interval: try: _max_age = int( getattr(self.config, "session_store_max_age_days", 0) or 0 ) if _max_age > 0: _pruned = self.session_store.prune_old_entries(_max_age) if _pruned: logger.info( "SessionStore prune: dropped %d stale entries", _pruned, ) except Exception as _e: logger.debug("SessionStore prune failed: %s", _e) self._last_session_store_prune_ts = time.time() except Exception as e: logger.debug("Session expiry watcher error: %s", e) # Sleep in small increments so we can stop quickly for _ in range(interval): if not self._running: break await asyncio.sleep(1) async def _kanban_notifier_watcher(self, interval: float = 5.0) -> None: """Poll ``kanban_notify_subs`` and deliver terminal events to users. For each subscription row, fetches ``task_events`` newer than the stored cursor with kind in the terminal set (``completed``, ``blocked``, ``gave_up``, ``crashed``, ``timed_out``). Sends one message per new event to ``(platform, chat_id, thread_id)``, then advances the cursor. When a task reaches a terminal state (``completed`` / ``archived``), the subscription is removed. Runs in the gateway event loop; all SQLite work is pushed to a thread via ``asyncio.to_thread`` so the loop never blocks on the WAL lock. Failures in one tick don't stop subsequent ticks. """ from gateway.config import Platform as _Platform try: from hermes_cli import kanban_db as _kb except Exception: logger.warning("kanban notifier: kanban_db not importable; notifier disabled") return TERMINAL_KINDS = ("completed", "blocked", "gave_up", "crashed", "timed_out") # Terminal event kinds trigger automatic unsubscription — the task # is done, blocked, or in a retry-needed state that the human # shouldn't keep pinging a stale chat for. Previously we only # unsubbed when task.status in ('done', 'archived'), which left # subscriptions on 'blocked' / 'gave_up' / 'crashed' / 'timed_out' # tasks stranded forever. TERMINAL_EVENT_KINDS = TERMINAL_KINDS # Per-subscription send-failure counter. Adapter.send raising # means the chat is dead (deleted, bot kicked, etc.) — after N # consecutive send failures the sub is dropped so we don't spin # against a dead chat every 5 seconds forever. MAX_SEND_FAILURES = 3 sub_fail_counts: dict[tuple, int] = getattr( self, "_kanban_sub_fail_counts", {} ) self._kanban_sub_fail_counts = sub_fail_counts # Initial delay so the gateway can finish wiring adapters. await asyncio.sleep(5) while self._running: try: def _collect(): conn = _kb.connect() try: _kb.init_db() # idempotent; handles first-run except Exception: pass try: subs = _kb.list_notify_subs(conn) deliveries: list[dict] = [] for sub in subs: cursor, events = _kb.unseen_events_for_sub( conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"], thread_id=sub.get("thread_id") or "", kinds=TERMINAL_KINDS, ) if not events: continue task = _kb.get_task(conn, sub["task_id"]) deliveries.append({ "sub": sub, "cursor": cursor, "events": events, "task": task, }) return deliveries finally: conn.close() deliveries = await asyncio.to_thread(_collect) for d in deliveries: sub = d["sub"] task = d["task"] platform_str = (sub["platform"] or "").lower() try: plat = _Platform(platform_str) except ValueError: # Unknown platform string; skip and advance cursor so # we don't replay forever. await asyncio.to_thread( self._kanban_advance, sub, d["cursor"], ) continue adapter = self.adapters.get(plat) if adapter is None: continue # platform not currently connected title = (task.title if task else sub["task_id"])[:120] for ev in d["events"]: kind = ev.kind # Identity prefix: attribute terminal pings to the # worker that did the work. Makes fleets (where one # chat subscribes to many tasks) legible at a glance. who = (task.assignee if task and task.assignee else None) tag = f"@{who} " if who else "" if kind == "completed": # Prefer the run's summary (the worker's # intentional human-facing handoff, carried # in the event payload), then fall back to # task.result for legacy rows written before # runs shipped. handoff = "" payload_summary = None if ev.payload and ev.payload.get("summary"): payload_summary = str(ev.payload["summary"]) if payload_summary: h = payload_summary.strip().splitlines()[0][:200] handoff = f"\n{h}" elif task and task.result: r = task.result.strip().splitlines()[0][:160] handoff = f"\n{r}" msg = ( f"✔ {tag}Kanban {sub['task_id']} done" f" — {title}{handoff}" ) elif kind == "blocked": reason = "" if ev.payload and ev.payload.get("reason"): reason = f": {str(ev.payload['reason'])[:160]}" msg = f"⏸ {tag}Kanban {sub['task_id']} blocked{reason}" elif kind == "gave_up": err = "" if ev.payload and ev.payload.get("error"): err = f"\n{str(ev.payload['error'])[:200]}" msg = ( f"✖ {tag}Kanban {sub['task_id']} gave up " f"after repeated spawn failures{err}" ) elif kind == "crashed": msg = ( f"✖ {tag}Kanban {sub['task_id']} worker crashed " f"(pid gone); dispatcher will retry" ) elif kind == "timed_out": limit = 0 if ev.payload and ev.payload.get("limit_seconds"): limit = int(ev.payload["limit_seconds"]) msg = ( f"⏱ {tag}Kanban {sub['task_id']} timed out " f"(max_runtime={limit}s); will retry" ) else: continue metadata: dict[str, Any] = {} if sub.get("thread_id"): metadata["thread_id"] = sub["thread_id"] sub_key = ( sub["task_id"], sub["platform"], sub["chat_id"], sub.get("thread_id") or "", ) try: await adapter.send( sub["chat_id"], msg, metadata=metadata, ) # Reset the failure counter on success. sub_fail_counts.pop(sub_key, None) except Exception as exc: fails = sub_fail_counts.get(sub_key, 0) + 1 sub_fail_counts[sub_key] = fails logger.warning( "kanban notifier: send failed for %s on %s " "(attempt %d/%d): %s", sub["task_id"], platform_str, fails, MAX_SEND_FAILURES, exc, ) if fails >= MAX_SEND_FAILURES: logger.warning( "kanban notifier: dropping subscription " "%s on %s after %d consecutive send failures", sub["task_id"], platform_str, fails, ) await asyncio.to_thread(self._kanban_unsub, sub) sub_fail_counts.pop(sub_key, None) # Don't advance cursor on send failure — retry next tick. break else: # All events delivered; advance cursor + maybe unsub. await asyncio.to_thread( self._kanban_advance, sub, d["cursor"], ) # Unsubscribe when the LAST delivered event is a # terminal kind (the task hit a "no further updates" # state), not just on task.status in {done, archived}. # Covers blocked / gave_up / crashed / timed_out which # used to leak subs forever. last_kind = d["events"][-1].kind if d["events"] else None task_terminal = task and task.status in ("done", "archived") event_terminal = last_kind in TERMINAL_EVENT_KINDS if task_terminal or event_terminal: await asyncio.to_thread( self._kanban_unsub, sub, ) except Exception as exc: logger.warning("kanban notifier tick failed: %s", exc) # Sleep with cancellation checks. for _ in range(int(max(1, interval))): if not self._running: return await asyncio.sleep(1) def _kanban_advance(self, sub: dict, cursor: int) -> None: """Sync helper: advance a subscription's cursor. Runs in to_thread.""" from hermes_cli import kanban_db as _kb conn = _kb.connect() try: _kb.advance_notify_cursor( conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"], thread_id=sub.get("thread_id") or "", new_cursor=cursor, ) finally: conn.close() def _kanban_unsub(self, sub: dict) -> None: from hermes_cli import kanban_db as _kb conn = _kb.connect() try: _kb.remove_notify_sub( conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"], thread_id=sub.get("thread_id") or "", ) finally: conn.close() async def _kanban_dispatcher_watcher(self) -> None: """Embedded kanban dispatcher — one tick every `dispatch_interval_seconds`. Gated by `kanban.dispatch_in_gateway` in config.yaml (default True). When true, the gateway hosts the single dispatcher for this profile: no separate `hermes kanban daemon` process needed. When false, the loop exits immediately and an external daemon is expected. Each tick calls :func:`kanban_db.dispatch_once` inside ``asyncio.to_thread`` so the SQLite WAL lock never blocks the event loop. Failures in one tick don't stop subsequent ticks — same pattern as `_kanban_notifier_watcher`. Shutdown: the loop checks ``self._running`` between ticks; gateway stop() flips it to False and cancels pending tasks, and the in-flight ``to_thread`` returns on its own after the current ``dispatch_once`` call finishes (typically <1ms on an idle board). """ # Read config once at boot. If the user flips the flag later, they # restart the gateway; same pattern as every other background # watcher here. Honours HERMES_KANBAN_DISPATCH_IN_GATEWAY env var # as an escape hatch (false-y value disables without editing YAML). try: from hermes_cli.config import load_config as _load_config except Exception: logger.warning("kanban dispatcher: config loader unavailable; disabled") return env_override = os.environ.get("HERMES_KANBAN_DISPATCH_IN_GATEWAY", "").strip().lower() if env_override in ("0", "false", "no", "off"): logger.info("kanban dispatcher: disabled via HERMES_KANBAN_DISPATCH_IN_GATEWAY env") return try: cfg = _load_config() except Exception as exc: logger.warning("kanban dispatcher: cannot load config (%s); disabled", exc) return kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} if not kanban_cfg.get("dispatch_in_gateway", True): logger.info( "kanban dispatcher: disabled via config kanban.dispatch_in_gateway=false" ) return try: from hermes_cli import kanban_db as _kb except Exception: logger.warning("kanban dispatcher: kanban_db not importable; dispatcher disabled") return interval = float(kanban_cfg.get("dispatch_interval_seconds", 60) or 60) if interval < 1.0: interval = 1.0 # sanity floor — tighter than this is a footgun # Initial delay so the gateway finishes wiring adapters before the # dispatcher spawns workers (those workers may hit gateway notify # subscriptions etc.). Matches the notifier watcher's delay. await asyncio.sleep(5) # Health telemetry mirrored from `_cmd_daemon`: warn when ready # queue is non-empty but spawns are 0 for N consecutive ticks — # usually means broken PATH, missing venv, or credential loss. HEALTH_WINDOW = 6 bad_ticks = 0 last_warn_at = 0 def _tick_once() -> "Optional[object]": """Run one dispatch_once; return result or None on error. Runs in a worker thread via `asyncio.to_thread`.""" conn = None try: conn = _kb.connect() try: _kb.init_db() # idempotent, handles first-run except Exception: pass return _kb.dispatch_once(conn) except Exception: logger.exception("kanban dispatcher: tick failed") return None finally: if conn is not None: try: conn.close() except Exception: pass def _ready_nonempty() -> bool: """Cheap probe: is there at least one ready+assigned+unclaimed task?""" conn = None try: conn = _kb.connect() row = conn.execute( "SELECT 1 FROM tasks " "WHERE status = 'ready' AND assignee IS NOT NULL " " AND claim_lock IS NULL LIMIT 1" ).fetchone() return row is not None except Exception: return False finally: if conn is not None: try: conn.close() except Exception: pass logger.info( "kanban dispatcher: embedded in gateway (interval=%.1fs)", interval ) while self._running: try: res = await asyncio.to_thread(_tick_once) if res is not None and getattr(res, "spawned", None): # Quiet by default — only log when something actually # happened, so an idle gateway stays silent. logger.info( "kanban dispatcher: tick spawned=%d reclaimed=%d " "crashed=%d timed_out=%d promoted=%d auto_blocked=%d", len(res.spawned), res.reclaimed, len(res.crashed) if hasattr(res.crashed, "__len__") else 0, len(res.timed_out) if hasattr(res.timed_out, "__len__") else 0, res.promoted, len(res.auto_blocked) if hasattr(res.auto_blocked, "__len__") else 0, ) # Health telemetry ready_pending = await asyncio.to_thread(_ready_nonempty) spawned_any = bool(res and getattr(res, "spawned", None)) if ready_pending and not spawned_any: bad_ticks += 1 else: bad_ticks = 0 if bad_ticks >= HEALTH_WINDOW: now = int(time.time()) if now - last_warn_at >= 300: logger.warning( "kanban dispatcher stuck: ready queue non-empty for " "%d consecutive ticks but 0 workers spawned. Check " "profile health (venv, PATH, credentials) and " "`hermes kanban list --status ready`.", bad_ticks, ) last_warn_at = now except asyncio.CancelledError: logger.debug("kanban dispatcher: cancelled") raise except Exception: logger.exception("kanban dispatcher: unexpected watcher error") # Sleep in 1s slices so shutdown is snappy — otherwise a stop() # waits up to `interval` seconds for the current sleep to finish. slept = 0.0 while slept < interval and self._running: await asyncio.sleep(min(1.0, interval - slept)) slept += 1.0 async def _platform_reconnect_watcher(self) -> None: """Background task that periodically retries connecting failed platforms. Uses exponential backoff: 30s → 60s → 120s → 240s → 300s (cap). Stops retrying a platform after 20 failed attempts or if the error is non-retryable (e.g. bad auth token). """ _MAX_ATTEMPTS = 20 _BACKOFF_CAP = 300 # 5 minutes max between retries await asyncio.sleep(10) # initial delay — let startup finish while self._running: if not self._failed_platforms: # Nothing to reconnect — sleep and check again for _ in range(30): if not self._running: return await asyncio.sleep(1) continue now = time.monotonic() for platform in list(self._failed_platforms.keys()): if not self._running: return info = self._failed_platforms[platform] if now < info["next_retry"]: continue # not time yet if info["attempts"] >= _MAX_ATTEMPTS: logger.warning( "Giving up reconnecting %s after %d attempts", platform.value, info["attempts"], ) del self._failed_platforms[platform] continue platform_config = info["config"] attempt = info["attempts"] + 1 logger.info( "Reconnecting %s (attempt %d/%d)...", platform.value, attempt, _MAX_ATTEMPTS, ) try: adapter = self._create_adapter(platform, platform_config) if not adapter: logger.warning( "Reconnect %s: adapter creation returned None, removing from retry queue", platform.value, ) del self._failed_platforms[platform] continue adapter.set_message_handler(self._handle_message) adapter.set_fatal_error_handler(self._handle_adapter_fatal_error) adapter.set_session_store(self.session_store) adapter.set_busy_session_handler(self._handle_active_session_busy_message) success = await self._connect_adapter_with_timeout(adapter, platform) if success: self.adapters[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) self.delivery_router.adapters = self.adapters del self._failed_platforms[platform] self._update_platform_runtime_status( platform.value, platform_state="connected", error_code=None, error_message=None, ) logger.info("✓ %s reconnected successfully", platform.value) # Rebuild channel directory with the new adapter try: from gateway.channel_directory import build_channel_directory await build_channel_directory(self.adapters) except Exception: pass else: # Check if the failure is non-retryable if adapter.has_fatal_error and not adapter.fatal_error_retryable: self._update_platform_runtime_status( platform.value, platform_state="fatal", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) logger.warning( "Reconnect %s: non-retryable error (%s), removing from retry queue", platform.value, adapter.fatal_error_message, ) del self._failed_platforms[platform] else: self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message or "failed to reconnect", ) backoff = min(30 * (2 ** (attempt - 1)), _BACKOFF_CAP) info["attempts"] = attempt info["next_retry"] = time.monotonic() + backoff logger.info( "Reconnect %s failed, next retry in %ds", platform.value, backoff, ) except Exception as e: self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=None, error_message=str(e), ) backoff = min(30 * (2 ** (attempt - 1)), _BACKOFF_CAP) info["attempts"] = attempt info["next_retry"] = time.monotonic() + backoff logger.warning( "Reconnect %s error: %s, next retry in %ds", platform.value, e, backoff, ) # Check every 10 seconds for platforms that need reconnection for _ in range(10): if not self._running: return await asyncio.sleep(1) async def stop( self, *, restart: bool = False, detached_restart: bool = False, service_restart: bool = False, ) -> None: """Stop the gateway and disconnect all adapters.""" if restart: self._restart_requested = True self._restart_detached = detached_restart self._restart_via_service = service_restart if self._stop_task is not None: await self._stop_task return async def _stop_impl() -> None: def _kill_tool_subprocesses(phase: str) -> None: """Kill tool subprocesses + tear down terminal envs + browsers. Called twice in the shutdown path: once eagerly after a drain timeout forces agent interrupt (so we reclaim bash/ sleep children before systemd TimeoutStopSec escalates to SIGKILL on the cgroup — #8202), and once as a final catch-all at the end of _stop_impl() for the graceful path or anything respawned mid-teardown. All steps are best-effort; exceptions are swallowed so one subsystem's failure doesn't block the rest. """ try: from tools.process_registry import process_registry _killed = process_registry.kill_all() if _killed: logger.info( "Shutdown (%s): killed %d tool subprocess(es)", phase, _killed, ) except Exception as _e: logger.debug("process_registry.kill_all (%s) error: %s", phase, _e) try: from tools.terminal_tool import cleanup_all_environments cleanup_all_environments() except Exception as _e: logger.debug("cleanup_all_environments (%s) error: %s", phase, _e) try: from tools.browser_tool import cleanup_all_browsers cleanup_all_browsers() except Exception as _e: logger.debug("cleanup_all_browsers (%s) error: %s", phase, _e) logger.info( "Stopping gateway%s...", " for restart" if self._restart_requested else "", ) self._running = False self._draining = True # Notify all chats with active agents BEFORE draining. # Adapters are still connected here, so messages can be sent. await self._notify_active_sessions_of_shutdown() timeout = self._restart_drain_timeout active_agents, timed_out = await self._drain_active_agents(timeout) if timed_out: logger.warning( "Gateway drain timed out after %.1fs with %d active agent(s); interrupting remaining work.", timeout, self._running_agent_count(), ) # Mark forcibly-interrupted sessions as resume_pending BEFORE # interrupting the agents. This preserves each session's # session_id + transcript so the next message on the same # session_key auto-resumes from the existing conversation # instead of getting routed through suspend_recently_active() # and converted into a fresh session. Terminal escalation # for genuinely stuck sessions still flows through the # existing ``.restart_failure_counts`` stuck-loop counter # (incremented below, threshold 3), which sets # ``suspended=True`` and overrides resume_pending. # # Iterate self._running_agents (current) rather than the # drain-start ``active_agents`` snapshot — the snapshot # may include sessions that finished gracefully during # the drain window, and marking those falsely would give # them a stray restart-interruption system note on their # next turn even though their previous turn completed # cleanly. Skip pending sentinels for the same reason # _interrupt_running_agents() does: their agent hasn't # started yet, there's nothing to interrupt, and the # session shouldn't carry a misleading resume flag. _resume_reason = ( "restart_timeout" if self._restart_requested else "shutdown_timeout" ) for _sk, _agent in list(self._running_agents.items()): if _agent is _AGENT_PENDING_SENTINEL: continue try: self.session_store.mark_resume_pending(_sk, _resume_reason) except Exception as _e: logger.debug( "mark_resume_pending failed for %s: %s", _sk, _e, ) self._interrupt_running_agents( _INTERRUPT_REASON_GATEWAY_RESTART if self._restart_requested else _INTERRUPT_REASON_GATEWAY_SHUTDOWN ) interrupt_deadline = asyncio.get_running_loop().time() + 5.0 while self._running_agents and asyncio.get_running_loop().time() < interrupt_deadline: self._update_runtime_status("draining") await asyncio.sleep(0.1) # Kill lingering tool subprocesses NOW, before we spend more # budget on adapter disconnect / session DB close. Under # systemd (TimeoutStopSec bounded by drain_timeout+headroom), # deferring this to the end of stop() risks systemd escalating # to SIGKILL on the cgroup first — at which point bash/sleep # children left behind by an interrupted terminal tool get # killed by systemd instead of us (issue #8202). The final # catch-all cleanup below still runs for the graceful path. _kill_tool_subprocesses("post-interrupt") if self._restart_requested and self._restart_detached: try: await self._launch_detached_restart_command() except Exception as e: logger.error("Failed to launch detached gateway restart: %s", e) self._finalize_shutdown_agents(active_agents) # Also shut down memory providers on idle cached agents. # _finalize_shutdown_agents only handles agents that were # mid-turn at drain time; the _agent_cache may still hold # idle agents whose MemoryProviders never received # on_session_end(). _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock is not None and _cache is not None: with _cache_lock: _idle_agents = list(_cache.values()) _cache.clear() for _entry in _idle_agents: _agent = ( _entry[0] if isinstance(_entry, tuple) else _entry ) self._cleanup_agent_resources(_agent) for platform, adapter in list(self.adapters.items()): try: await adapter.cancel_background_tasks() except Exception as e: logger.debug("✗ %s background-task cancel error: %s", platform.value, e) try: await adapter.disconnect() logger.info("✓ %s disconnected", platform.value) except Exception as e: logger.error("✗ %s disconnect error: %s", platform.value, e) for _task in list(self._background_tasks): if _task is self._stop_task: continue _task.cancel() self._background_tasks.clear() self.adapters.clear() self._running_agents.clear() self._running_agents_ts.clear() self._pending_messages.clear() self._pending_approvals.clear() if hasattr(self, '_busy_ack_ts'): self._busy_ack_ts.clear() self._shutdown_event.set() # Global cleanup: kill any remaining tool subprocesses not tied # to a specific agent (catch-all for zombie prevention). On the # drain-timeout path we already did this earlier after agent # interrupt — this second call catches (a) the graceful path # where drain succeeded without interrupt, and (b) anything # that got respawned between the earlier call and adapter # disconnect (defense in depth; safe to call repeatedly). _kill_tool_subprocesses("final-cleanup") # Reap the process-global auxiliary-client cache once at the very # end of teardown. Per-turn cleanup runs in _cleanup_agent_resources # for each active agent, but clients bound to worker-thread loops # that died with their ThreadPoolExecutor (notably cron ticks) only # get swept here. Without this, long-running gateways accumulate # async httpx transports until they hit EMFILE on macOS's default # RLIMIT_NOFILE=256. See #14210. try: from agent.auxiliary_client import shutdown_cached_clients shutdown_cached_clients() except Exception as _e: logger.debug("shutdown_cached_clients error: %s", _e) # Close SQLite session DBs so the WAL write lock is released. # Without this, --replace and similar restart flows leave the # old gateway's connection holding the WAL lock until Python # actually exits — causing 'database is locked' errors when # the new gateway tries to open the same file. for _db_holder in (self, getattr(self, "session_store", None)): _db = getattr(_db_holder, "_db", None) if _db_holder else None if _db is None or not hasattr(_db, "close"): continue try: _db.close() except Exception as _e: logger.debug("SessionDB close error: %s", _e) from gateway.status import remove_pid_file, release_gateway_runtime_lock remove_pid_file() release_gateway_runtime_lock() # Write a clean-shutdown marker so the next startup knows this # wasn't a crash. suspend_recently_active() only needs to run # after unexpected exits. However, if the drain timed out and # agents were force-interrupted, their sessions may be in an # incomplete state (trailing tool response, no final assistant # message). Skip the marker in that case so the next startup # suspends those sessions — giving users a clean slate instead # of resuming a half-finished tool loop. if not timed_out: try: (_hermes_home / ".clean_shutdown").touch() except Exception: pass else: logger.info( "Skipping .clean_shutdown marker — drain timed out with " "interrupted agents; next startup will suspend recently " "active sessions." ) # Track sessions that were active at shutdown for stuck-loop # detection (#7536). On each restart, the counter increments # for sessions that were running. If a session hits the # threshold (3 consecutive restarts while active), the next # startup auto-suspends it — breaking the loop. if active_agents: self._increment_restart_failure_counts(set(active_agents.keys())) if self._restart_requested and self._restart_via_service: self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE self._exit_reason = self._exit_reason or "Gateway restart requested" self._draining = False self._update_runtime_status("stopped", self._exit_reason) logger.info("Gateway stopped") self._stop_task = asyncio.create_task(_stop_impl()) await self._stop_task async def wait_for_shutdown(self) -> None: """Wait for shutdown signal.""" await self._shutdown_event.wait() def _create_adapter( self, platform: Platform, config: Any ) -> Optional[BasePlatformAdapter]: """Create the appropriate adapter for a platform. Checks the platform_registry first (plugin adapters), then falls through to the built-in if/elif chain for core platforms. """ if hasattr(config, "extra") and isinstance(config.extra, dict): config.extra.setdefault( "group_sessions_per_user", self.config.group_sessions_per_user, ) config.extra.setdefault( "thread_sessions_per_user", getattr(self.config, "thread_sessions_per_user", False), ) # ── Plugin-registered platforms (checked first) ─────────────────── try: from gateway.platform_registry import platform_registry if platform_registry.is_registered(platform.value): adapter = platform_registry.create_adapter(platform.value, config) if adapter is not None: return adapter # Registered but failed to instantiate — don't silently fall # through to built-ins (there are none for plugin platforms). logger.error( "Platform '%s' is registered but adapter creation failed " "(check dependencies and config)", platform.value, ) return None except Exception as e: logger.debug("Platform registry lookup for '%s' failed: %s", platform.value, e) # Fall through to built-in adapters below if platform == Platform.TELEGRAM: from gateway.platforms.telegram import TelegramAdapter, check_telegram_requirements if not check_telegram_requirements(): logger.warning("Telegram: python-telegram-bot not installed") return None return TelegramAdapter(config) elif platform == Platform.DISCORD: from gateway.platforms.discord import DiscordAdapter, check_discord_requirements if not check_discord_requirements(): logger.warning("Discord: discord.py not installed") return None adapter = DiscordAdapter(config) adapter.gateway_runner = self # For cross-platform admin alerts on unauthorized slash return adapter elif platform == Platform.WHATSAPP: from gateway.platforms.whatsapp import WhatsAppAdapter, check_whatsapp_requirements if not check_whatsapp_requirements(): logger.warning("WhatsApp: Node.js not installed or bridge not configured") return None return WhatsAppAdapter(config) elif platform == Platform.SLACK: from gateway.platforms.slack import SlackAdapter, check_slack_requirements if not check_slack_requirements(): logger.warning("Slack: slack-bolt not installed. Run: pip install 'hermes-agent[slack]'") return None return SlackAdapter(config) elif platform == Platform.SIGNAL: from gateway.platforms.signal import SignalAdapter, check_signal_requirements if not check_signal_requirements(): logger.warning("Signal: SIGNAL_HTTP_URL or SIGNAL_ACCOUNT not configured") return None return SignalAdapter(config) elif platform == Platform.HOMEASSISTANT: from gateway.platforms.homeassistant import HomeAssistantAdapter, check_ha_requirements if not check_ha_requirements(): logger.warning("HomeAssistant: aiohttp not installed or HASS_TOKEN not set") return None return HomeAssistantAdapter(config) elif platform == Platform.EMAIL: from gateway.platforms.email import EmailAdapter, check_email_requirements if not check_email_requirements(): logger.warning("Email: EMAIL_ADDRESS, EMAIL_PASSWORD, EMAIL_IMAP_HOST, or EMAIL_SMTP_HOST not set") return None return EmailAdapter(config) elif platform == Platform.SMS: from gateway.platforms.sms import SmsAdapter, check_sms_requirements if not check_sms_requirements(): logger.warning("SMS: aiohttp not installed or TWILIO_ACCOUNT_SID/TWILIO_AUTH_TOKEN not set") return None return SmsAdapter(config) elif platform == Platform.DINGTALK: from gateway.platforms.dingtalk import DingTalkAdapter, check_dingtalk_requirements if not check_dingtalk_requirements(): logger.warning("DingTalk: dingtalk-stream not installed or DINGTALK_CLIENT_ID/SECRET not set") return None return DingTalkAdapter(config) elif platform == Platform.FEISHU: from gateway.platforms.feishu import FeishuAdapter, check_feishu_requirements if not check_feishu_requirements(): logger.warning("Feishu: lark-oapi not installed or FEISHU_APP_ID/SECRET not set") return None return FeishuAdapter(config) elif platform == Platform.WECOM_CALLBACK: from gateway.platforms.wecom_callback import ( WecomCallbackAdapter, check_wecom_callback_requirements, ) if not check_wecom_callback_requirements(): logger.warning("WeComCallback: aiohttp/httpx not installed") return None return WecomCallbackAdapter(config) elif platform == Platform.WECOM: from gateway.platforms.wecom import WeComAdapter, check_wecom_requirements if not check_wecom_requirements(): logger.warning("WeCom: aiohttp not installed or WECOM_BOT_ID/SECRET not set") return None return WeComAdapter(config) elif platform == Platform.WEIXIN: from gateway.platforms.weixin import WeixinAdapter, check_weixin_requirements if not check_weixin_requirements(): logger.warning("Weixin: aiohttp/cryptography not installed") return None return WeixinAdapter(config) elif platform == Platform.MATTERMOST: from gateway.platforms.mattermost import MattermostAdapter, check_mattermost_requirements if not check_mattermost_requirements(): logger.warning("Mattermost: MATTERMOST_TOKEN or MATTERMOST_URL not set, or aiohttp missing") return None return MattermostAdapter(config) elif platform == Platform.MATRIX: from gateway.platforms.matrix import MatrixAdapter, check_matrix_requirements if not check_matrix_requirements(): logger.warning("Matrix: mautrix not installed or credentials not set. Run: pip install 'mautrix[encryption]'") return None return MatrixAdapter(config) elif platform == Platform.API_SERVER: from gateway.platforms.api_server import APIServerAdapter, check_api_server_requirements if not check_api_server_requirements(): logger.warning("API Server: aiohttp not installed") return None return APIServerAdapter(config) elif platform == Platform.WEBHOOK: from gateway.platforms.webhook import WebhookAdapter, check_webhook_requirements if not check_webhook_requirements(): logger.warning("Webhook: aiohttp not installed") return None adapter = WebhookAdapter(config) adapter.gateway_runner = self # For cross-platform delivery return adapter elif platform == Platform.BLUEBUBBLES: from gateway.platforms.bluebubbles import BlueBubblesAdapter, check_bluebubbles_requirements if not check_bluebubbles_requirements(): logger.warning("BlueBubbles: aiohttp/httpx missing or BLUEBUBBLES_SERVER_URL/BLUEBUBBLES_PASSWORD not configured") return None return BlueBubblesAdapter(config) elif platform == Platform.QQBOT: from gateway.platforms.qqbot import QQAdapter, check_qq_requirements if not check_qq_requirements(): logger.warning("QQBot: aiohttp/httpx missing or QQ_APP_ID/QQ_CLIENT_SECRET not configured") return None return QQAdapter(config) elif platform == Platform.YUANBAO: from gateway.platforms.yuanbao import YuanbaoAdapter, WEBSOCKETS_AVAILABLE if not WEBSOCKETS_AVAILABLE: logger.warning("Yuanbao: websockets not installed. Run: pip install websockets") return None return YuanbaoAdapter(config) return None def _is_user_authorized(self, source: SessionSource) -> bool: """ Check if a user is authorized to use the bot. Checks in order: 1. Per-platform allow-all flag (e.g., DISCORD_ALLOW_ALL_USERS=true) 2. Environment variable allowlists (TELEGRAM_ALLOWED_USERS, etc.) 3. DM pairing approved list 4. Global allow-all (GATEWAY_ALLOW_ALL_USERS=true) 5. Default: deny """ # Home Assistant events are system-generated (state changes), not # user-initiated messages. The HASS_TOKEN already authenticates the # connection, so HA events are always authorized. # Webhook events are authenticated via HMAC signature validation in # the adapter itself — no user allowlist applies. if source.platform in (Platform.HOMEASSISTANT, Platform.WEBHOOK): return True user_id = source.user_id if not user_id: return False platform_env_map = { Platform.TELEGRAM: "TELEGRAM_ALLOWED_USERS", Platform.DISCORD: "DISCORD_ALLOWED_USERS", Platform.WHATSAPP: "WHATSAPP_ALLOWED_USERS", Platform.SLACK: "SLACK_ALLOWED_USERS", Platform.SIGNAL: "SIGNAL_ALLOWED_USERS", Platform.EMAIL: "EMAIL_ALLOWED_USERS", Platform.SMS: "SMS_ALLOWED_USERS", Platform.MATTERMOST: "MATTERMOST_ALLOWED_USERS", Platform.MATRIX: "MATRIX_ALLOWED_USERS", Platform.DINGTALK: "DINGTALK_ALLOWED_USERS", Platform.FEISHU: "FEISHU_ALLOWED_USERS", Platform.WECOM: "WECOM_ALLOWED_USERS", Platform.WECOM_CALLBACK: "WECOM_CALLBACK_ALLOWED_USERS", Platform.WEIXIN: "WEIXIN_ALLOWED_USERS", Platform.BLUEBUBBLES: "BLUEBUBBLES_ALLOWED_USERS", Platform.QQBOT: "QQ_ALLOWED_USERS", Platform.YUANBAO: "YUANBAO_ALLOWED_USERS", } platform_group_user_env_map = { Platform.TELEGRAM: "TELEGRAM_GROUP_ALLOWED_USERS", } platform_group_chat_env_map = { Platform.TELEGRAM: "TELEGRAM_GROUP_ALLOWED_CHATS", Platform.QQBOT: "QQ_GROUP_ALLOWED_USERS", } platform_allow_all_map = { Platform.TELEGRAM: "TELEGRAM_ALLOW_ALL_USERS", Platform.DISCORD: "DISCORD_ALLOW_ALL_USERS", Platform.WHATSAPP: "WHATSAPP_ALLOW_ALL_USERS", Platform.SLACK: "SLACK_ALLOW_ALL_USERS", Platform.SIGNAL: "SIGNAL_ALLOW_ALL_USERS", Platform.EMAIL: "EMAIL_ALLOW_ALL_USERS", Platform.SMS: "SMS_ALLOW_ALL_USERS", Platform.MATTERMOST: "MATTERMOST_ALLOW_ALL_USERS", Platform.MATRIX: "MATRIX_ALLOW_ALL_USERS", Platform.DINGTALK: "DINGTALK_ALLOW_ALL_USERS", Platform.FEISHU: "FEISHU_ALLOW_ALL_USERS", Platform.WECOM: "WECOM_ALLOW_ALL_USERS", Platform.WECOM_CALLBACK: "WECOM_CALLBACK_ALLOW_ALL_USERS", Platform.WEIXIN: "WEIXIN_ALLOW_ALL_USERS", Platform.BLUEBUBBLES: "BLUEBUBBLES_ALLOW_ALL_USERS", Platform.QQBOT: "QQ_ALLOW_ALL_USERS", Platform.YUANBAO: "YUANBAO_ALLOW_ALL_USERS", } # Bots admitted by {PLATFORM}_ALLOW_BOTS bypass the human allowlist (#4466). platform_allow_bots_map = { Platform.DISCORD: "DISCORD_ALLOW_BOTS", Platform.FEISHU: "FEISHU_ALLOW_BOTS", } # Plugin platforms: check the registry for auth env var names if source.platform not in platform_env_map: try: from gateway.platform_registry import platform_registry entry = platform_registry.get(source.platform.value) if entry: if entry.allowed_users_env: platform_env_map[source.platform] = entry.allowed_users_env if entry.allow_all_env: platform_allow_all_map[source.platform] = entry.allow_all_env except Exception: pass # Per-platform allow-all flag (e.g., DISCORD_ALLOW_ALL_USERS=true) platform_allow_all_var = platform_allow_all_map.get(source.platform, "") if platform_allow_all_var and os.getenv(platform_allow_all_var, "").lower() in ("true", "1", "yes"): return True if getattr(source, "is_bot", False): allow_bots_var = platform_allow_bots_map.get(source.platform) if allow_bots_var and os.getenv(allow_bots_var, "none").lower().strip() in ("mentions", "all"): return True # Discord role-based access (DISCORD_ALLOWED_ROLES): the adapter's # on_message pre-filter already verified role membership — if the # message reached here, the user passed that check. Authorize # directly to avoid the "no allowlists configured" branch below # rejecting role-only setups where DISCORD_ALLOWED_USERS is empty # (issue #7871). if ( source.platform == Platform.DISCORD and os.getenv("DISCORD_ALLOWED_ROLES", "").strip() ): return True # Check pairing store (always checked, regardless of allowlists) platform_name = source.platform.value if source.platform else "" if self.pairing_store.is_approved(platform_name, user_id): return True # Check platform-specific and global allowlists platform_allowlist = os.getenv(platform_env_map.get(source.platform, ""), "").strip() group_user_allowlist = "" group_chat_allowlist = "" if source.chat_type in {"group", "forum"}: group_user_allowlist = os.getenv(platform_group_user_env_map.get(source.platform, ""), "").strip() group_chat_allowlist = os.getenv(platform_group_chat_env_map.get(source.platform, ""), "").strip() global_allowlist = os.getenv("GATEWAY_ALLOWED_USERS", "").strip() if not platform_allowlist and not group_user_allowlist and not group_chat_allowlist and not global_allowlist: # No allowlists configured -- check global allow-all flag return os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in ("true", "1", "yes") # Telegram can optionally authorize group traffic by chat ID. # Keep this separate from TELEGRAM_GROUP_ALLOWED_USERS, which gates # the sender user ID for group/forum messages. if group_chat_allowlist and source.chat_type in {"group", "forum"} and source.chat_id: allowed_group_ids = { chat_id.strip() for chat_id in group_chat_allowlist.split(",") if chat_id.strip() } if "*" in allowed_group_ids or source.chat_id in allowed_group_ids: return True # Backward-compat shim for #15027: prior to PR #17686, # TELEGRAM_GROUP_ALLOWED_USERS was (mis)used as a chat-ID allowlist. # Values starting with "-" are Telegram chat IDs, not user IDs, so if # users still have those in TELEGRAM_GROUP_ALLOWED_USERS we honor them # as chat IDs and warn once. The correct var is now # TELEGRAM_GROUP_ALLOWED_CHATS. if ( source.platform == Platform.TELEGRAM and group_user_allowlist and source.chat_type in {"group", "forum"} and source.chat_id ): legacy_chat_ids = { v.strip() for v in group_user_allowlist.split(",") if v.strip().startswith("-") } if legacy_chat_ids: if not getattr(self, "_warned_telegram_group_users_legacy", False): logger.warning( "TELEGRAM_GROUP_ALLOWED_USERS contains chat-ID-shaped values " "(%s). Treating them as chat IDs for backward compatibility. " "Move chat IDs to TELEGRAM_GROUP_ALLOWED_CHATS — the _USERS var " "is now for sender user IDs.", ",".join(sorted(legacy_chat_ids)), ) self._warned_telegram_group_users_legacy = True if source.chat_id in legacy_chat_ids: return True # Check if user is in any allowlist. In group/forum chats, # TELEGRAM_GROUP_ALLOWED_USERS is the scoped allowlist and should not # imply DM access; TELEGRAM_ALLOWED_USERS remains the platform-wide # allowlist and still works everywhere for backward compatibility. allowed_ids = set() if platform_allowlist: allowed_ids.update(uid.strip() for uid in platform_allowlist.split(",") if uid.strip()) if group_user_allowlist: allowed_ids.update(uid.strip() for uid in group_user_allowlist.split(",") if uid.strip()) if global_allowlist: allowed_ids.update(uid.strip() for uid in global_allowlist.split(",") if uid.strip()) # "*" in any allowlist means allow everyone (consistent with # SIGNAL_GROUP_ALLOWED_USERS precedent) if "*" in allowed_ids: return True check_ids = {user_id} if "@" in user_id: check_ids.add(user_id.split("@")[0]) # WhatsApp: resolve phone↔LID aliases from bridge session mapping files if source.platform == Platform.WHATSAPP: normalized_allowed_ids = set() for allowed_id in allowed_ids: normalized_allowed_ids.update(_expand_whatsapp_auth_aliases(allowed_id)) if normalized_allowed_ids: allowed_ids = normalized_allowed_ids check_ids.update(_expand_whatsapp_auth_aliases(user_id)) normalized_user_id = _normalize_whatsapp_identifier(user_id) if normalized_user_id: check_ids.add(normalized_user_id) return bool(check_ids & allowed_ids) def _get_unauthorized_dm_behavior(self, platform: Optional[Platform]) -> str: """Return how unauthorized DMs should be handled for a platform. Resolution order: 1. Explicit per-platform ``unauthorized_dm_behavior`` in config — always wins. 2. Explicit global ``unauthorized_dm_behavior`` in config — wins when no per-platform. 3. When an allowlist (``PLATFORM_ALLOWED_USERS``, ``PLATFORM_GROUP_ALLOWED_USERS`` / ``PLATFORM_GROUP_ALLOWED_CHATS``, or ``GATEWAY_ALLOWED_USERS``) is configured, default to ``"ignore"`` — the allowlist signals that the owner has deliberately restricted access; spamming unknown contacts with pairing codes is both noisy and a potential info-leak. (#9337) 4. No allowlist and no explicit config → ``"pair"`` (open-gateway default). """ config = getattr(self, "config", None) # Check for an explicit per-platform override first. if config and hasattr(config, "get_unauthorized_dm_behavior") and platform: platform_cfg = config.platforms.get(platform) if hasattr(config, "platforms") else None if platform_cfg and "unauthorized_dm_behavior" in getattr(platform_cfg, "extra", {}): # Operator explicitly configured behavior for this platform — respect it. return config.get_unauthorized_dm_behavior(platform) # Check for an explicit global config override. if config and hasattr(config, "unauthorized_dm_behavior"): if config.unauthorized_dm_behavior != "pair": # non-default → explicit override return config.unauthorized_dm_behavior # No explicit override. Fall back to allowlist-aware default: # if any allowlist is configured for this platform, silently drop # unauthorized messages instead of sending pairing codes. if platform: platform_env_map = { Platform.TELEGRAM: "TELEGRAM_ALLOWED_USERS", Platform.DISCORD: "DISCORD_ALLOWED_USERS", Platform.WHATSAPP: "WHATSAPP_ALLOWED_USERS", Platform.SLACK: "SLACK_ALLOWED_USERS", Platform.SIGNAL: "SIGNAL_ALLOWED_USERS", Platform.EMAIL: "EMAIL_ALLOWED_USERS", Platform.SMS: "SMS_ALLOWED_USERS", Platform.MATTERMOST: "MATTERMOST_ALLOWED_USERS", Platform.MATRIX: "MATRIX_ALLOWED_USERS", Platform.DINGTALK: "DINGTALK_ALLOWED_USERS", Platform.FEISHU: "FEISHU_ALLOWED_USERS", Platform.WECOM: "WECOM_ALLOWED_USERS", Platform.WECOM_CALLBACK: "WECOM_CALLBACK_ALLOWED_USERS", Platform.WEIXIN: "WEIXIN_ALLOWED_USERS", Platform.BLUEBUBBLES: "BLUEBUBBLES_ALLOWED_USERS", Platform.QQBOT: "QQ_ALLOWED_USERS", } platform_group_env_map = { Platform.TELEGRAM: ( "TELEGRAM_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_CHATS", ), Platform.QQBOT: ("QQ_GROUP_ALLOWED_USERS",), } if os.getenv(platform_env_map.get(platform, ""), "").strip(): return "ignore" for env_key in platform_group_env_map.get(platform, ()): if os.getenv(env_key, "").strip(): return "ignore" if os.getenv("GATEWAY_ALLOWED_USERS", "").strip(): return "ignore" return "pair" async def _deliver_platform_notice(self, source, content: str) -> None: """Deliver a setup/operational notice using platform-specific privacy rules.""" adapter = self.adapters.get(source.platform) if not adapter: return config = getattr(self, "config", None) notice_delivery = "public" if config and hasattr(config, "get_notice_delivery"): notice_delivery = config.get_notice_delivery(source.platform) metadata = {"thread_id": source.thread_id} if getattr(source, "thread_id", None) else None if notice_delivery == "private" and getattr(source, "user_id", None): try: result = await adapter.send_private_notice( source.chat_id, source.user_id, content, metadata=metadata, ) if getattr(result, "success", False): return except Exception: logger.debug( "[%s] send_private_notice failed, falling back to public", getattr(source, "platform", "?"), exc_info=True, ) await adapter.send(source.chat_id, content, metadata=metadata) async def _handle_message(self, event: MessageEvent) -> Optional[str]: """ Handle an incoming message from any platform. This is the core message processing pipeline: 1. Check user authorization 2. Check for commands (/new, /reset, etc.) 3. Check for running agent and interrupt if needed 4. Get or create session 5. Build context for agent 6. Run agent conversation 7. Return response """ source = event.source # Stale-code self-check (Issue #17648). A gateway that survives # ``hermes update`` keeps old modules cached in sys.modules; the # first inbound message is our earliest safe chance to detect # this and restart gracefully before we dispatch to the agent # and hit ImportError on freshly-added names (e.g. cfg_get). # Idempotent — runs the real check at most once per message, and # request_restart() no-ops after the first call. try: if self._detect_stale_code(): self._trigger_stale_code_restart() # Acknowledge to the user so they don't see a silent # drop; the gateway will be back up in a moment via the # service manager / profile-watcher respawn. return ( "⟳ Gateway code was updated in the background — " "restarting this gateway so your next message runs " "on the new code. Please retry in a moment." ) except Exception as _stale_exc: logger.debug("Stale-code self-check failed: %s", _stale_exc) # Internal events (e.g. background-process completion notifications) # are system-generated and must skip user authorization. is_internal = bool(getattr(event, "internal", False)) # Fire pre_gateway_dispatch plugin hook for user-originated messages. # Plugins receive the MessageEvent and may return a dict influencing flow: # {"action": "skip", "reason": ...} -> drop (no reply, plugin handled) # {"action": "rewrite", "text": ...} -> replace event.text, continue # {"action": "allow"} / None -> normal dispatch # Hook runs BEFORE auth so plugins can handle unauthorized senders # (e.g. customer handover ingest) without triggering the pairing flow. if not is_internal: try: from hermes_cli.plugins import invoke_hook as _invoke_hook _hook_results = _invoke_hook( "pre_gateway_dispatch", event=event, gateway=self, session_store=self.session_store, ) except Exception as _hook_exc: logger.warning("pre_gateway_dispatch invocation failed: %s", _hook_exc) _hook_results = [] for _result in _hook_results: if not isinstance(_result, dict): continue _action = _result.get("action") if _action == "skip": logger.info( "pre_gateway_dispatch skip: reason=%s platform=%s chat=%s", _result.get("reason"), source.platform.value if source.platform else "unknown", source.chat_id or "unknown", ) return None if _action == "rewrite": _new_text = _result.get("text") if isinstance(_new_text, str): event = dataclasses.replace(event, text=_new_text) source = event.source break if _action == "allow": break if is_internal: pass elif source.user_id is None: # Messages with no user identity (Telegram service messages, # channel forwards, anonymous admin actions) cannot be # authorized — drop silently instead of triggering the pairing # flow with a None user_id. logger.debug("Ignoring message with no user_id from %s", source.platform.value) return None elif not self._is_user_authorized(source): logger.warning("Unauthorized user: %s (%s) on %s", source.user_id, source.user_name, source.platform.value) # In DMs: offer pairing code. In groups: silently ignore. if source.chat_type == "dm" and self._get_unauthorized_dm_behavior(source.platform) == "pair": platform_name = source.platform.value if source.platform else "unknown" # Rate-limit ALL pairing responses (code or rejection) to # prevent spamming the user with repeated messages when # multiple DMs arrive in quick succession. if self.pairing_store._is_rate_limited(platform_name, source.user_id): return None code = self.pairing_store.generate_code( platform_name, source.user_id, source.user_name or "" ) if code: adapter = self.adapters.get(source.platform) if adapter: await adapter.send( source.chat_id, f"Hi~ I don't recognize you yet!\n\n" f"Here's your pairing code: `{code}`\n\n" f"Ask the bot owner to run:\n" f"`hermes pairing approve {platform_name} {code}`" ) else: adapter = self.adapters.get(source.platform) if adapter: await adapter.send( source.chat_id, "Too many pairing requests right now~ " "Please try again later!" ) # Record rate limit so subsequent messages are silently ignored self.pairing_store._record_rate_limit(platform_name, source.user_id) return None # Intercept messages that are responses to a pending /update prompt. # The update process (detached) wrote .update_prompt.json; the watcher # forwarded it to the user; now the user's reply goes back via # .update_response so the update process can continue. # # IMPORTANT: recognized slash commands must bypass this interception. # Otherwise control/session commands like /new or /help get silently # consumed as update answers instead of being dispatched normally. _quick_key = self._session_key_for_source(source) _update_prompts = getattr(self, "_update_prompt_pending", {}) if _update_prompts.get(_quick_key): raw = (event.text or "").strip() # Accept /approve and /deny as shorthand for yes/no cmd = event.get_command() if cmd in ("approve", "yes"): response_text = "y" elif cmd in ("deny", "no"): response_text = "n" else: _recognized_cmd = None if cmd: try: from hermes_cli.commands import resolve_command as _resolve_update_cmd except Exception: _resolve_update_cmd = None if _resolve_update_cmd is not None: try: _cmd_def = _resolve_update_cmd(cmd) _recognized_cmd = _cmd_def.name if _cmd_def else None except Exception: _recognized_cmd = None if _recognized_cmd: response_text = "" else: response_text = raw if response_text: response_path = _hermes_home / ".update_response" try: tmp = response_path.with_suffix(".tmp") tmp.write_text(response_text) tmp.replace(response_path) except OSError as e: logger.warning("Failed to write update response: %s", e) return f"✗ Failed to send response to update process: {e}" _update_prompts.pop(_quick_key, None) label = response_text if len(response_text) <= 20 else response_text[:20] + "…" return f"✓ Sent `{label}` to the update process." # Recognized slash command during a pending update prompt: # unblock the detached update subprocess by writing a blank # response so ``_gateway_prompt`` returns the prompt's default # (typically a safe "n" / skip) and exits cleanly instead of # blocking on stdin until the 30-minute watcher timeout. # The slash command then falls through to normal dispatch. if _recognized_cmd: response_path = _hermes_home / ".update_response" try: tmp = response_path.with_suffix(".tmp") tmp.write_text("") tmp.replace(response_path) logger.info( "Recognized /%s during pending update prompt for %s; " "cancelled prompt with default and dispatching command", _recognized_cmd, _quick_key, ) except OSError as e: logger.warning( "Failed to write cancel response for pending update prompt: %s", e, ) _update_prompts.pop(_quick_key, None) # Intercept messages that are responses to a pending /reload-mcp # (or future) slash-confirm prompt. Recognized confirm replies are # /approve, /always, /cancel (plus short aliases). Anything else # falls through to normal dispatch — a stale pending confirm does # NOT block other commands. # # Important: if a dangerous-command approval is ALSO pending (agent # blocked inside tools/approval.py), the tool approval takes # precedence — /approve there unblocks the waiting tool thread. # Slash-confirm only catches /approve when no tool approval is live. from tools import slash_confirm as _slash_confirm_mod _pending_confirm = _slash_confirm_mod.get_pending(_quick_key) _tool_approval_live = False try: from tools.approval import has_blocking_approval _tool_approval_live = has_blocking_approval(_quick_key) except Exception: _tool_approval_live = False if _pending_confirm and not _tool_approval_live: _raw_reply = (event.text or "").strip() _cmd_reply = event.get_command() _confirm_choice = None if _cmd_reply in ("approve", "yes", "ok", "confirm"): _confirm_choice = "once" elif _cmd_reply in ("always", "remember"): _confirm_choice = "always" elif _cmd_reply in ("cancel", "no", "deny", "nevermind"): _confirm_choice = "cancel" elif _raw_reply.lower() in ("approve", "approve once", "once"): _confirm_choice = "once" elif _raw_reply.lower() in ("always", "always approve"): _confirm_choice = "always" elif _raw_reply.lower() in ("cancel", "nevermind", "no"): _confirm_choice = "cancel" if _confirm_choice is not None: _resolved = await _slash_confirm_mod.resolve( _quick_key, _pending_confirm.get("confirm_id"), _confirm_choice, ) return _resolved or "" # Stale pending + unrelated command: drop the pending state so # the confirm doesn't block normal usage indefinitely. The user # clearly moved on. _slash_confirm_mod.clear_if_stale(_quick_key) # PRIORITY handling when an agent is already running for this session. # Default behavior is to interrupt immediately so user text/stop messages # are handled with minimal latency. # # Special case: Telegram/photo bursts often arrive as multiple near- # simultaneous updates. Do NOT interrupt for photo-only follow-ups here; # let the adapter-level batching/queueing logic absorb them. # Staleness eviction: detect leaked locks from hung/crashed handlers. # With inactivity-based timeout, active tasks can run for hours, so # wall-clock age alone isn't sufficient. Evict only when the agent # has been *idle* beyond the inactivity threshold (or when the agent # object has no activity tracker and wall-clock age is extreme). _raw_stale_timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) _stale_ts = self._running_agents_ts.get(_quick_key, 0) if _quick_key in self._running_agents and _stale_ts: _stale_age = time.time() - _stale_ts _stale_agent = self._running_agents.get(_quick_key) # Never evict the pending sentinel — it was just placed moments # ago during the async setup phase before the real agent is # created. Sentinels have no get_activity_summary(), so the # idle check below would always evaluate to inf >= timeout and # immediately evict them, racing with the setup path. _stale_idle = float("inf") # assume idle if we can't check _stale_detail = "" if _stale_agent and hasattr(_stale_agent, "get_activity_summary"): try: _sa = _stale_agent.get_activity_summary() _stale_idle = _sa.get("seconds_since_activity", float("inf")) _stale_detail = ( f" | last_activity={_sa.get('last_activity_desc', 'unknown')} " f"({_stale_idle:.0f}s ago) " f"| iteration={_sa.get('api_call_count', 0)}/{_sa.get('max_iterations', 0)}" ) except Exception: pass # Evict if: agent is idle beyond timeout, OR wall-clock age is # extreme (10x timeout or 2h, whichever is larger — catches # cases where the agent object was garbage-collected). _wall_ttl = max(_raw_stale_timeout * 10, 7200) if _raw_stale_timeout > 0 else float("inf") _should_evict = ( _stale_agent is not _AGENT_PENDING_SENTINEL and ( (_raw_stale_timeout > 0 and _stale_idle >= _raw_stale_timeout) or _stale_age > _wall_ttl ) ) if _should_evict: logger.warning( "Evicting stale _running_agents entry for %s " "(age: %.0fs, idle: %.0fs, timeout: %.0fs)%s", _quick_key, _stale_age, _stale_idle, _raw_stale_timeout, _stale_detail, ) self._invalidate_session_run_generation( _quick_key, reason="stale_running_agent_eviction", ) self._release_running_agent_state(_quick_key) if _quick_key in self._running_agents: if event.get_command() == "status": return await self._handle_status_command(event) # Resolve the command once for all early-intercept checks below. from hermes_cli.commands import ( ACTIVE_SESSION_BYPASS_COMMANDS as _DEDICATED_HANDLERS, resolve_command as _resolve_cmd_inner, ) _evt_cmd = event.get_command() _cmd_def_inner = _resolve_cmd_inner(_evt_cmd) if _evt_cmd else None if _cmd_def_inner and _cmd_def_inner.name == "restart": return await self._handle_restart_command(event) # /stop must hard-kill the session when an agent is running. # A soft interrupt (agent.interrupt()) doesn't help when the agent # is truly hung — the executor thread is blocked and never checks # _interrupt_requested. Force-clean _running_agents so the session # is unlocked and subsequent messages are processed normally. if _cmd_def_inner and _cmd_def_inner.name == "stop": await self._interrupt_and_clear_session( _quick_key, source, interrupt_reason=_INTERRUPT_REASON_STOP, invalidation_reason="stop_command", ) logger.info("STOP for session %s — agent interrupted, session lock released", _quick_key) return EphemeralReply("⚡ Stopped. You can continue this session.") # /reset and /new must bypass the running-agent guard so they # actually dispatch as commands instead of being queued as user # text (which would be fed back to the agent with the same # broken history — #2170). Interrupt the agent first, then # clear the adapter's pending queue so the stale "/reset" text # doesn't get re-processed as a user message after the # interrupt completes. if _cmd_def_inner and _cmd_def_inner.name == "new": # Clear any pending messages so the old text doesn't replay await self._interrupt_and_clear_session( _quick_key, source, interrupt_reason=_INTERRUPT_REASON_RESET, invalidation_reason="new_command", ) # Clean up the running agent entry so the reset handler # doesn't think an agent is still active. return await self._handle_reset_command(event) # /queue — queue without interrupting. # Semantics: each /queue invocation produces its own full agent # turn, processed in FIFO order after the current run (and any # earlier /queue items) finishes. Messages are NOT merged. if event.get_command() in ("queue", "q"): queued_text = event.get_command_args().strip() if not queued_text: return "Usage: /queue " adapter = self.adapters.get(source.platform) if adapter: queued_event = MessageEvent( text=queued_text, message_type=MessageType.TEXT, source=event.source, message_id=event.message_id, channel_prompt=event.channel_prompt, ) self._enqueue_fifo(_quick_key, queued_event, adapter) depth = self._queue_depth(_quick_key, adapter=self.adapters.get(source.platform)) if depth <= 1: return "Queued for the next turn." return f"Queued for the next turn. ({depth} queued)" # /steer — inject mid-run after the next tool call. # Unlike /queue (turn boundary), /steer lands BETWEEN tool-call # iterations inside the same agent run, by appending to the # last tool result's content. No interrupt, no new user turn, # no role-alternation violation. if _cmd_def_inner and _cmd_def_inner.name == "steer": steer_text = event.get_command_args().strip() if not steer_text: return "Usage: /steer " running_agent = self._running_agents.get(_quick_key) if running_agent is _AGENT_PENDING_SENTINEL: # Agent hasn't started yet — queue as turn-boundary fallback. adapter = self.adapters.get(source.platform) if adapter: queued_event = MessageEvent( text=steer_text, message_type=MessageType.TEXT, source=event.source, message_id=event.message_id, channel_prompt=event.channel_prompt, ) adapter._pending_messages[_quick_key] = queued_event return "Agent still starting — /steer queued for the next turn." if running_agent and hasattr(running_agent, "steer"): try: accepted = running_agent.steer(steer_text) except Exception as exc: logger.warning("Steer failed for session %s: %s", _quick_key, exc) return f"⚠️ Steer failed: {exc}" if accepted: preview = steer_text[:60] + ("..." if len(steer_text) > 60 else "") return f"⏩ Steer queued — arrives after the next tool call: '{preview}'" return "Steer rejected (empty payload)." # Running agent is missing or lacks steer() — fall back to queue. adapter = self.adapters.get(source.platform) if adapter: queued_event = MessageEvent( text=steer_text, message_type=MessageType.TEXT, source=event.source, message_id=event.message_id, channel_prompt=event.channel_prompt, ) adapter._pending_messages[_quick_key] = queued_event return "No active agent — /steer queued for the next turn." # /model must not be used while the agent is running. if _cmd_def_inner and _cmd_def_inner.name == "model": return "Agent is running — wait or /stop first, then switch models." # /approve and /deny must bypass the running-agent interrupt path. # The agent thread is blocked on a threading.Event inside # tools/approval.py — sending an interrupt won't unblock it. # Route directly to the approval handler so the event is signalled. if _cmd_def_inner and _cmd_def_inner.name in ("approve", "deny"): if _cmd_def_inner.name == "approve": return await self._handle_approve_command(event) return await self._handle_deny_command(event) # /agents (/tasks alias) should be query-only and never interrupt. if _cmd_def_inner and _cmd_def_inner.name == "agents": return await self._handle_agents_command(event) # /background must bypass the running-agent guard — it starts a # parallel task and must never interrupt the active conversation. # /btw is an alias of /background and resolves to the same canonical # name, so this branch handles both commands. if _cmd_def_inner and _cmd_def_inner.name == "background": return await self._handle_background_command(event) # /kanban must bypass the guard. It writes to a profile-agnostic # DB (kanban.db), not to the running agent's state. In fact # /kanban unblock is often the only way to free a worker that # has blocked waiting for a peer — letting that be dispatched # mid-run is the whole point of the board. if _cmd_def_inner and _cmd_def_inner.name == "kanban": return await self._handle_kanban_command(event) # /goal is safe mid-run for status/pause/clear (inspection and # control-plane only — doesn't interrupt the running turn). # Setting a new goal text mid-run is rejected with the same # "wait or /stop" message as /model so we don't race a second # continuation prompt against the current turn. if _cmd_def_inner and _cmd_def_inner.name == "goal": _goal_arg = (event.get_command_args() or "").strip().lower() if not _goal_arg or _goal_arg in ("status", "pause", "resume", "clear", "stop", "done"): return await self._handle_goal_command(event) return "Agent is running — use /goal status / pause / clear mid-run, or /stop before setting a new goal." # Session-level toggles that are safe to run mid-agent — # /yolo can unblock a pending approval prompt, /verbose cycles # the tool-progress display mode for the ongoing stream. # Both modify session state without needing agent interaction # and must not be queued (the safety net would discard them). # /fast and /reasoning are config-only and take effect next # message, so they fall through to the catch-all busy response # below — users should wait and set them between turns. if _cmd_def_inner and _cmd_def_inner.name in ("yolo", "verbose"): if _cmd_def_inner.name == "yolo": return await self._handle_yolo_command(event) if _cmd_def_inner.name == "verbose": return await self._handle_verbose_command(event) if _cmd_def_inner.name == "footer": return await self._handle_footer_command(event) # Gateway-handled info/control commands with dedicated # running-agent handlers. if _cmd_def_inner and _cmd_def_inner.name in _DEDICATED_HANDLERS: if _cmd_def_inner.name == "help": return await self._handle_help_command(event) if _cmd_def_inner.name == "commands": return await self._handle_commands_command(event) if _cmd_def_inner.name == "profile": return await self._handle_profile_command(event) if _cmd_def_inner.name == "update": return await self._handle_update_command(event) # Catch-all: any other recognized slash command reached the # running-agent guard. Reject gracefully rather than falling # through to interrupt + discard. Without this, commands # like /model, /reasoning, /voice, /insights, /title, # /resume, /retry, /undo, /compress, /usage, # /reload-mcp, /sethome, /reset (all registered as Discord # slash commands) would interrupt the agent AND get # silently discarded by the slash-command safety net, # producing a zero-char response. See #5057, #6252, #10370. if _cmd_def_inner: return ( f"⏳ Agent is running — `/{_cmd_def_inner.name}` can't run " f"mid-turn. Wait for the current response or `/stop` first." ) if event.message_type == MessageType.PHOTO: logger.debug("PRIORITY photo follow-up for session %s — queueing without interrupt", _quick_key) adapter = self.adapters.get(source.platform) if adapter: merge_pending_message_event(adapter._pending_messages, _quick_key, event) return None _telegram_followup_grace = float( os.getenv("HERMES_TELEGRAM_FOLLOWUP_GRACE_SECONDS", "3.0") ) _started_at = self._running_agents_ts.get(_quick_key, 0) if ( source.platform == Platform.TELEGRAM and event.message_type == MessageType.TEXT and _telegram_followup_grace > 0 and _started_at and (time.time() - _started_at) <= _telegram_followup_grace ): logger.debug( "Telegram follow-up arrived %.2fs after run start for %s — queueing without interrupt", time.time() - _started_at, _quick_key, ) adapter = self.adapters.get(source.platform) if adapter: merge_pending_message_event( adapter._pending_messages, _quick_key, event, merge_text=True, ) return None running_agent = self._running_agents.get(_quick_key) if running_agent is _AGENT_PENDING_SENTINEL: # Agent is being set up but not ready yet. if event.get_command() == "stop": # Force-clean the sentinel so the session is unlocked. self._release_running_agent_state(_quick_key) logger.info("HARD STOP (pending) for session %s — sentinel cleared", _quick_key) return EphemeralReply("⚡ Force-stopped. The agent was still starting — session unlocked.") # Queue the message so it will be picked up after the # agent starts. adapter = self.adapters.get(source.platform) if adapter: merge_pending_message_event( adapter._pending_messages, _quick_key, event, merge_text=True, ) return None if self._draining: if self._queue_during_drain_enabled(): self._queue_or_replace_pending_event(_quick_key, event) return ( f"⏳ Gateway {self._status_action_gerund()} — queued for the next turn after it comes back." if self._queue_during_drain_enabled() else f"⏳ Gateway is {self._status_action_gerund()} and is not accepting another turn right now." ) if self._busy_input_mode == "queue": logger.debug("PRIORITY queue follow-up for session %s", _quick_key) self._queue_or_replace_pending_event(_quick_key, event) return None if self._busy_input_mode == "steer": # Steer mode: inject text into the running agent mid-run via # agent.steer(). Falls back to queue semantics if the payload # is empty, the agent lacks steer(), or steer() rejects. steer_text = (event.text or "").strip() steered = False if steer_text and hasattr(running_agent, "steer"): try: steered = bool(running_agent.steer(steer_text)) except Exception as exc: logger.warning("PRIORITY steer failed for session %s: %s", _quick_key, exc) steered = False if steered: logger.debug("PRIORITY steer for session %s", _quick_key) return None logger.debug("PRIORITY steer-fallback-to-queue for session %s", _quick_key) self._queue_or_replace_pending_event(_quick_key, event) return None logger.debug("PRIORITY interrupt for session %s", _quick_key) running_agent.interrupt(event.text) if _quick_key in self._pending_messages: self._pending_messages[_quick_key] += "\n" + event.text else: self._pending_messages[_quick_key] = event.text return None # Check for commands command = event.get_command() from hermes_cli.commands import ( GATEWAY_KNOWN_COMMANDS, is_gateway_known_command, resolve_command as _resolve_cmd, ) # Resolve aliases to canonical name so dispatch and hook names # don't depend on the exact alias the user typed. _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command # Fire the ``command:`` hook for any recognized slash # command — built-in OR plugin-registered. Handlers can return a # dict with ``{"decision": "deny" | "handled" | "rewrite", ...}`` # to intercept dispatch before core handling runs. This replaces # the previous fire-and-forget emit(): return values are now # honored, but handlers that return nothing behave exactly as # before (telemetry-style hooks keep working). if command and is_gateway_known_command(canonical): raw_args = event.get_command_args().strip() hook_ctx = { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "command": canonical, "raw_command": command, "args": raw_args, "raw_args": raw_args, } try: hook_results = await self.hooks.emit_collect( f"command:{canonical}", hook_ctx ) except Exception as _hook_err: logger.debug( "command:%s hook dispatch failed (non-fatal): %s", canonical, _hook_err, ) hook_results = [] for hook_result in hook_results: if not isinstance(hook_result, dict): continue decision = str(hook_result.get("decision", "")).strip().lower() if not decision or decision == "allow": continue if decision == "deny": message = hook_result.get("message") if isinstance(message, str) and message: return message return f"Command `/{command}` was blocked by a hook." if decision == "handled": message = hook_result.get("message") return message if isinstance(message, str) and message else None if decision == "rewrite": new_command = str( hook_result.get("command_name", "") ).strip().lstrip("/") if not new_command: continue new_args = str(hook_result.get("raw_args", "")).strip() event.text = f"/{new_command} {new_args}".strip() command = event.get_command() _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command break if canonical == "new": return await self._handle_reset_command(event) if canonical == "help": return await self._handle_help_command(event) if canonical == "commands": return await self._handle_commands_command(event) if canonical == "profile": return await self._handle_profile_command(event) if canonical == "status": return await self._handle_status_command(event) if canonical == "agents": return await self._handle_agents_command(event) if canonical == "restart": return await self._handle_restart_command(event) if canonical == "stop": return await self._handle_stop_command(event) if canonical == "reasoning": return await self._handle_reasoning_command(event) if canonical == "fast": return await self._handle_fast_command(event) if canonical == "verbose": return await self._handle_verbose_command(event) if canonical == "footer": return await self._handle_footer_command(event) if canonical == "yolo": return await self._handle_yolo_command(event) if canonical == "model": return await self._handle_model_command(event) if canonical == "personality": return await self._handle_personality_command(event) if canonical == "kanban": return await self._handle_kanban_command(event) if canonical == "retry": return await self._handle_retry_command(event) if canonical == "undo": return await self._handle_undo_command(event) if canonical == "sethome": return await self._handle_set_home_command(event) if canonical == "compress": return await self._handle_compress_command(event) if canonical == "usage": return await self._handle_usage_command(event) if canonical == "insights": return await self._handle_insights_command(event) if canonical == "reload-mcp": return await self._handle_reload_mcp_command(event) if canonical == "reload-skills": return await self._handle_reload_skills_command(event) if canonical == "approve": return await self._handle_approve_command(event) if canonical == "deny": return await self._handle_deny_command(event) if canonical == "update": return await self._handle_update_command(event) if canonical == "debug": return await self._handle_debug_command(event) if canonical == "title": return await self._handle_title_command(event) if canonical == "resume": return await self._handle_resume_command(event) if canonical == "branch": return await self._handle_branch_command(event) if canonical == "rollback": return await self._handle_rollback_command(event) if canonical == "background": return await self._handle_background_command(event) if canonical == "steer": # No active agent — /steer has no tool call to inject into. # Strip the prefix so downstream treats it as a normal user # message. If the payload is empty, surface the usage hint. steer_payload = event.get_command_args().strip() if not steer_payload: return "Usage: /steer (no agent is running; sending as a normal message)" try: event.text = steer_payload except Exception: pass # Do NOT return — fall through to _handle_message_with_agent # at the end of this function so the rewritten text is sent # to the agent as a regular user turn. if canonical == "goal": return await self._handle_goal_command(event) if canonical == "voice": return await self._handle_voice_command(event) if self._draining: return f"⏳ Gateway is {self._status_action_gerund()} and is not accepting new work right now." # User-defined quick commands (bypass agent loop, no LLM call) if command: if isinstance(self.config, dict): quick_commands = self.config.get("quick_commands", {}) or {} else: quick_commands = getattr(self.config, "quick_commands", {}) or {} if not isinstance(quick_commands, dict): quick_commands = {} if command in quick_commands: qcmd = quick_commands[command] if qcmd.get("type") == "exec": exec_cmd = qcmd.get("command", "") if exec_cmd: try: proc = await asyncio.create_subprocess_shell( exec_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) output = (stdout or stderr).decode().strip() return output if output else "Command returned no output." except asyncio.TimeoutError: return "Quick command timed out (30s)." except Exception as e: return f"Quick command error: {e}" else: return f"Quick command '/{command}' has no command defined." elif qcmd.get("type") == "alias": target = qcmd.get("target", "").strip() if target: target = target if target.startswith("/") else f"/{target}" target_command = target.lstrip("/") user_args = event.get_command_args().strip() event.text = f"{target} {user_args}".strip() command = target_command # Fall through to normal command dispatch below else: return f"Quick command '/{command}' has no target defined." else: return f"Quick command '/{command}' has unsupported type (supported: 'exec', 'alias')." # Plugin-registered slash commands if command: try: from hermes_cli.plugins import get_plugin_command_handler # Normalize underscores to hyphens so Telegram's underscored # autocomplete form matches plugin commands registered with # hyphens. See hermes_cli/commands.py:_build_telegram_menu. plugin_handler = get_plugin_command_handler(command.replace("_", "-")) if plugin_handler: user_args = event.get_command_args().strip() result = plugin_handler(user_args) if asyncio.iscoroutine(result): result = await result return str(result) if result else None except Exception as e: logger.debug("Plugin command dispatch failed (non-fatal): %s", e) # Skill slash commands: /skill-name loads the skill and sends to agent. # resolve_skill_command_key() handles the Telegram underscore/hyphen # round-trip so /claude_code from Telegram autocomplete still resolves # to the claude-code skill. if command: try: from agent.skill_commands import ( get_skill_commands, build_skill_invocation_message, resolve_skill_command_key, ) skill_cmds = get_skill_commands() cmd_key = resolve_skill_command_key(command) if cmd_key is not None: # Check per-platform disabled status before executing. # get_skill_commands() only applies the *global* disabled # list at scan time; per-platform overrides need checking # here because the cache is process-global across platforms. _skill_name = skill_cmds[cmd_key].get("name", "") _plat = source.platform.value if source.platform else None if _plat and _skill_name: from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled if _skill_name in _get_plat_disabled(platform=_plat): return ( f"The **{_skill_name}** skill is disabled for {_plat}.\n" f"Enable it with: `hermes skills config`" ) user_instruction = event.get_command_args().strip() msg = build_skill_invocation_message( cmd_key, user_instruction, task_id=_quick_key ) if msg: event.text = msg # Fall through to normal message processing with skill content else: # Not an active skill — check if it's a known-but-disabled or # uninstalled skill and give actionable guidance. _unavail_msg = _check_unavailable_skill(command) if _unavail_msg: return _unavail_msg # Genuinely unrecognized /command: not a built-in, not a # plugin, not a skill, not a known-inactive skill. Warn # the user instead of silently forwarding it to the LLM # as free text (which leads to silent-failure behavior # like the model inventing a delegate_task call). # Normalize to hyphenated form before checking known # built-ins (command may be an alias target set by the # quick-command block above, so _cmd_def can be stale). if command.replace("_", "-") not in GATEWAY_KNOWN_COMMANDS: logger.warning( "Unrecognized slash command /%s from %s — " "replying with unknown-command notice", command, source.platform.value if source.platform else "?", ) return ( f"Unknown command `/{command}`. " f"Type /commands to see what's available, " f"or resend without the leading slash to send " f"as a regular message." ) except Exception as e: logger.debug("Skill command check failed (non-fatal): %s", e) # Pending exec approvals are handled by /approve and /deny commands above. # No bare text matching — "yes" in normal conversation must not trigger # execution of a dangerous command. # ── Claim this session before any await ─────────────────────── # Between here and _run_agent registering the real AIAgent, there # are numerous await points (hooks, vision enrichment, STT, # session hygiene compression). Without this sentinel a second # message arriving during any of those yields would pass the # "already running" guard and spin up a duplicate agent for the # same session — corrupting the transcript. self._running_agents[_quick_key] = _AGENT_PENDING_SENTINEL self._running_agents_ts[_quick_key] = time.time() _run_generation = self._begin_session_run_generation(_quick_key) try: _agent_result = await self._handle_message_with_agent(event, source, _quick_key, _run_generation) # Goal continuation: after the agent returns a final response # for this turn, check any standing /goal — the judge will # either mark it done, pause it (budget), or enqueue a # continuation prompt back through the adapter FIFO so the # next turn makes more progress. Wrapped in try/except so a # broken judge never breaks normal message handling. try: _final_text = "" if isinstance(_agent_result, dict): _final_text = str(_agent_result.get("final_response") or "") elif isinstance(_agent_result, str): _final_text = _agent_result # Skip for empty responses (interrupted / errored) — the # judge would almost always say "continue" and we'd loop # on error. Let the user drive the next turn. if _final_text.strip(): try: session_entry = self.session_store.get_or_create_session(source) except Exception: session_entry = None if session_entry is not None: self._post_turn_goal_continuation( session_entry=session_entry, source=source, final_response=_final_text, ) except Exception as _goal_exc: logger.debug("goal continuation hook failed: %s", _goal_exc) return _agent_result finally: # If _run_agent replaced the sentinel with a real agent and # then cleaned it up, this is a no-op. If we exited early # (exception, command fallthrough, etc.) the sentinel must # not linger or the session would be permanently locked out. if self._running_agents.get(_quick_key) is _AGENT_PENDING_SENTINEL: self._release_running_agent_state(_quick_key) else: # Agent path already cleaned _running_agents; make sure # the paired metadata dicts are gone too. self._running_agents_ts.pop(_quick_key, None) if hasattr(self, "_busy_ack_ts"): self._busy_ack_ts.pop(_quick_key, None) async def _prepare_inbound_message_text( self, *, event: MessageEvent, source: SessionSource, history: List[Dict[str, Any]], ) -> Optional[str]: """Prepare inbound event text for the agent. Keep the normal inbound path and the queued follow-up path on the same preprocessing pipeline so sender attribution, image enrichment, STT, document notes, reply context, and @ references all behave the same. Side effect: buffers per-session native image paths when the active model supports native vision AND the user has images attached. The caller consumes and clears that session-scoped buffer at the ``run_conversation`` site to build a multimodal user turn. When the list is empty, the ``_enrich_message_with_vision`` text path has already run and images are represented in-text. """ history = history or [] message_text = event.text or "" _group_sessions_per_user = getattr(self.config, "group_sessions_per_user", True) _thread_sessions_per_user = getattr(self.config, "thread_sessions_per_user", False) # Use the same helper every other call site uses so the write key here # matches the consume key at the run_conversation site — even if the # session store overrides build_session_key's default behavior. session_key = self._session_key_for_source(source) # Reset only this session's per-call buffer; other sessions may be # concurrently preparing multimodal turns on the same runner. self._consume_pending_native_image_paths(session_key) _is_shared_multi_user = is_shared_multi_user_session( source, group_sessions_per_user=_group_sessions_per_user, thread_sessions_per_user=_thread_sessions_per_user, ) if _is_shared_multi_user and source.user_name: message_text = f"[{source.user_name}] {message_text}" if event.media_urls: image_paths = [] audio_paths = [] for i, path in enumerate(event.media_urls): mtype = event.media_types[i] if i < len(event.media_types) else "" if mtype.startswith("image/") or event.message_type == MessageType.PHOTO: image_paths.append(path) if mtype.startswith("audio/") or event.message_type in (MessageType.VOICE, MessageType.AUDIO): audio_paths.append(path) if image_paths: # Decide routing: native (attach pixels) vs text (vision_analyze # pre-run + prepend description). See agent/image_routing.py. _img_mode = self._decide_image_input_mode() if _img_mode == "native": # Defer attachment to the run_conversation call site. pending_native = getattr(self, "_pending_native_image_paths_by_session", None) if pending_native is None: pending_native = {} self._pending_native_image_paths_by_session = pending_native pending_native[session_key] = list(image_paths) logger.info( "Image routing: native (model supports vision). %d image(s) will be attached inline.", len(image_paths), ) else: logger.info( "Image routing: text (mode=%s). Pre-analyzing %d image(s) via vision_analyze.", _img_mode, len(image_paths), ) message_text = await self._enrich_message_with_vision( message_text, image_paths, ) if audio_paths: message_text = await self._enrich_message_with_transcription( message_text, audio_paths, ) _stt_fail_markers = ( "No STT provider", "STT is disabled", "can't listen", "VOICE_TOOLS_OPENAI_KEY", ) if any(marker in message_text for marker in _stt_fail_markers): _stt_adapter = self.adapters.get(source.platform) _stt_meta = {"thread_id": source.thread_id} if source.thread_id else None if _stt_adapter: try: _stt_msg = ( "🎤 I received your voice message but can't transcribe it — " "no speech-to-text provider is configured.\n\n" "To enable voice: install faster-whisper " "(`pip install faster-whisper` in the Hermes venv) " "and set `stt.enabled: true` in config.yaml, " "then /restart the gateway." ) if self._has_setup_skill(): _stt_msg += "\n\nFor full setup instructions, type: `/skill hermes-agent-setup`" await _stt_adapter.send( source.chat_id, _stt_msg, metadata=_stt_meta, ) except Exception: pass if event.media_urls and event.message_type == MessageType.DOCUMENT: import mimetypes as _mimetypes _TEXT_EXTENSIONS = {".txt", ".md", ".csv", ".log", ".json", ".xml", ".yaml", ".yml", ".toml", ".ini", ".cfg"} for i, path in enumerate(event.media_urls): mtype = event.media_types[i] if i < len(event.media_types) else "" if mtype in ("", "application/octet-stream"): _ext = os.path.splitext(path)[1].lower() if _ext in _TEXT_EXTENSIONS: mtype = "text/plain" else: guessed, _ = _mimetypes.guess_type(path) if guessed: mtype = guessed if not mtype.startswith(("application/", "text/")): continue basename = os.path.basename(path) parts = basename.split("_", 2) display_name = parts[2] if len(parts) >= 3 else basename display_name = re.sub(r'[^\w.\- ]', '_', display_name) if mtype.startswith("text/"): context_note = ( f"[The user sent a text document: '{display_name}'. " f"Its content has been included below. " f"The file is also saved at: {path}]" ) else: context_note = ( f"[The user sent a document: '{display_name}'. " f"The file is saved at: {path}. " f"Ask the user what they'd like you to do with it.]" ) message_text = f"{context_note}\n\n{message_text}" if getattr(event, "reply_to_text", None) and event.reply_to_message_id: # Always inject the reply-to pointer — even when the quoted text # already appears in history. The prefix isn't deduplication, it's # disambiguation: it tells the agent *which* prior message the user # is referencing. History can contain the same or similar text # multiple times, and without an explicit pointer the agent has to # guess (or answer for both subjects). Token overhead is minimal. reply_snippet = event.reply_to_text[:500] message_text = f'[Replying to: "{reply_snippet}"]\n\n{message_text}' if "@" in message_text: try: from agent.context_references import preprocess_context_references_async from agent.model_metadata import get_model_context_length _msg_cwd = os.environ.get("TERMINAL_CWD", os.path.expanduser("~")) _msg_runtime = _resolve_runtime_agent_kwargs() _msg_config_ctx = None try: _msg_cfg = _load_gateway_config() _msg_model_cfg = _msg_cfg.get("model", {}) if isinstance(_msg_model_cfg, dict): _msg_raw_ctx = _msg_model_cfg.get("context_length") if _msg_raw_ctx is not None: _msg_config_ctx = int(_msg_raw_ctx) except Exception: pass _msg_ctx_len = get_model_context_length( self._model, base_url=self._base_url or _msg_runtime.get("base_url") or "", api_key=_msg_runtime.get("api_key") or "", config_context_length=_msg_config_ctx, ) _ctx_result = await preprocess_context_references_async( message_text, cwd=_msg_cwd, context_length=_msg_ctx_len, allowed_root=_msg_cwd, ) if _ctx_result.blocked: _adapter = self.adapters.get(source.platform) if _adapter: await _adapter.send( source.chat_id, "\n".join(_ctx_result.warnings) or "Context injection refused.", ) return None if _ctx_result.expanded: message_text = _ctx_result.message except Exception as exc: logger.debug("@ context reference expansion failed: %s", exc) return message_text def _consume_pending_native_image_paths(self, session_key: str) -> List[str]: pending_native = getattr(self, "_pending_native_image_paths_by_session", None) if not pending_native: return [] return list(pending_native.pop(session_key, []) or []) async def _handle_message_with_agent(self, event, source, _quick_key: str, run_generation: int): """Inner handler that runs under the _running_agents sentinel guard.""" _msg_start_time = time.time() _platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform) _msg_preview = (event.text or "")[:80].replace("\n", " ") logger.info( "inbound message: platform=%s user=%s chat=%s msg=%r", _platform_name, source.user_name or source.user_id or "unknown", source.chat_id or "unknown", _msg_preview, ) # Get or create session session_entry = self.session_store.get_or_create_session(source) session_key = session_entry.session_key if getattr(session_entry, "was_auto_reset", False): # Treat auto-reset as a full conversation boundary — drop every # session-scoped transient state so the fresh session does not # inherit the previous conversation's model/reasoning overrides # or a queued "/model switched" note. self._session_model_overrides.pop(session_key, None) self._set_session_reasoning_override(session_key, None) if hasattr(self, "_pending_model_notes"): self._pending_model_notes.pop(session_key, None) # Emit session:start for new or auto-reset sessions _is_new_session = ( session_entry.created_at == session_entry.updated_at or getattr(session_entry, "was_auto_reset", False) or getattr(session_entry, "is_fresh_reset", False) ) # Consume the is_fresh_reset flag immediately so it doesn't leak # onto subsequent messages in the same session (issue #6508). if getattr(session_entry, "is_fresh_reset", False): session_entry.is_fresh_reset = False if _is_new_session: await self.hooks.emit("session:start", { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "session_id": session_entry.session_id, "session_key": session_key, }) # Build session context context = build_session_context(source, self.config, session_entry) # Set session context variables for tools (task-local, concurrency-safe) _session_env_tokens = self._set_session_env(context) # Read privacy.redact_pii from config (re-read per message) _redact_pii = False try: _pcfg = _load_gateway_config() _redact_pii = bool((_pcfg.get("privacy") or {}).get("redact_pii", False)) except Exception: pass # Build the context prompt to inject context_prompt = build_session_context_prompt(context, redact_pii=_redact_pii) # If the previous session expired and was auto-reset, prepend a notice # so the agent knows this is a fresh conversation (not an intentional /reset). if getattr(session_entry, 'was_auto_reset', False): reset_reason = getattr(session_entry, 'auto_reset_reason', None) or 'idle' if reset_reason == "suspended": context_note = "[System note: The user's previous session was stopped and suspended. This is a fresh conversation with no prior context.]" elif reset_reason == "daily": context_note = "[System note: The user's session was automatically reset by the daily schedule. This is a fresh conversation with no prior context.]" else: context_note = "[System note: The user's previous session expired due to inactivity. This is a fresh conversation with no prior context.]" context_prompt = context_note + "\n\n" + context_prompt # Send a user-facing notification explaining the reset, unless: # - notifications are disabled in config # - the platform is excluded (e.g. api_server, webhook) # - the expired session had no activity (nothing was cleared) try: policy = self.session_store.config.get_reset_policy( platform=source.platform, session_type=getattr(source, 'chat_type', 'dm'), ) platform_name = source.platform.value if source.platform else "" had_activity = getattr(session_entry, 'reset_had_activity', False) # Suspended sessions always notify (they were explicitly stopped # or crashed mid-operation) — skip the policy check. should_notify = reset_reason == "suspended" or ( policy.notify and had_activity and platform_name not in policy.notify_exclude_platforms ) if should_notify: adapter = self.adapters.get(source.platform) if adapter: if reset_reason == "suspended": reason_text = "previous session was stopped or interrupted" elif reset_reason == "daily": reason_text = f"daily schedule at {policy.at_hour}:00" else: hours = policy.idle_minutes // 60 mins = policy.idle_minutes % 60 duration = f"{hours}h" if not mins else f"{hours}h {mins}m" if hours else f"{mins}m" reason_text = f"inactive for {duration}" notice = ( f"◐ Session automatically reset ({reason_text}). " f"Conversation history cleared.\n" f"Use /resume to browse and restore a previous session.\n" f"Adjust reset timing in config.yaml under session_reset." ) try: session_info = self._format_session_info() if session_info: notice = f"{notice}\n\n{session_info}" except Exception: pass await adapter.send( source.chat_id, notice, metadata=getattr(event, 'metadata', None), ) except Exception as e: logger.debug("Auto-reset notification failed (non-fatal): %s", e) session_entry.was_auto_reset = False session_entry.auto_reset_reason = None # Auto-load skill(s) for topic/channel bindings (Telegram DM Topics, # Discord channel_skill_bindings). Supports a single name or ordered list. # Only inject on NEW sessions — ongoing conversations already have the # skill content in their conversation history from the first message. _auto = getattr(event, "auto_skill", None) if _is_new_session and _auto: _skill_names = [_auto] if isinstance(_auto, str) else list(_auto) try: from agent.skill_commands import _load_skill_payload, _build_skill_message _combined_parts: list[str] = [] _loaded_names: list[str] = [] for _sname in _skill_names: _loaded = _load_skill_payload(_sname, task_id=_quick_key) if _loaded: _loaded_skill, _skill_dir, _display_name = _loaded _note = ( f'[IMPORTANT: The "{_display_name}" skill is auto-loaded. ' f"Follow its instructions for this session.]" ) _part = _build_skill_message(_loaded_skill, _skill_dir, _note) if _part: _combined_parts.append(_part) _loaded_names.append(_sname) else: logger.warning("[Gateway] Auto-skill '%s' not found", _sname) if _combined_parts: # Append the user's original text after all skill payloads _combined_parts.append(event.text) event.text = "\n\n".join(_combined_parts) logger.info( "[Gateway] Auto-loaded skill(s) %s for session %s", _loaded_names, session_key, ) except Exception as e: logger.warning("[Gateway] Failed to auto-load skill(s) %s: %s", _skill_names, e) # Load conversation history from transcript history = self.session_store.load_transcript(session_entry.session_id) # ----------------------------------------------------------------- # Session hygiene: auto-compress pathologically large transcripts # # Long-lived gateway sessions can accumulate enough history that # every new message rehydrates an oversized transcript, causing # repeated truncation/context failures. Detect this early and # compress proactively — before the agent even starts. (#628) # # Token source priority: # 1. Actual API-reported prompt_tokens from the last turn # (stored in session_entry.last_prompt_tokens) # 2. Rough char-based estimate (str(msg)//4). Overestimates # by 30-50% on code/JSON-heavy sessions, but that just # means hygiene fires a bit early — safe and harmless. # ----------------------------------------------------------------- if history and len(history) >= 4: from agent.model_metadata import ( estimate_messages_tokens_rough, get_model_context_length, ) # Read model + compression config from config.yaml. # NOTE: hygiene threshold is intentionally HIGHER than the agent's # own compressor (0.85 vs 0.50). Hygiene is a safety net for # sessions that grew too large between turns — it fires pre-agent # to prevent API failures. The agent's own compressor handles # normal context management during its tool loop with accurate # real token counts. Having hygiene at 0.50 caused premature # compression on every turn in long gateway sessions. _hyg_model = "anthropic/claude-sonnet-4.6" _hyg_threshold_pct = 0.85 _hyg_compression_enabled = True _hyg_hard_msg_limit = 400 _hyg_config_context_length = None _hyg_provider = None _hyg_base_url = None _hyg_api_key = None _hyg_data = {} try: _hyg_data = _load_gateway_config() if _hyg_data: # Resolve model name (same logic as run_sync) _model_cfg = _hyg_data.get("model", {}) if isinstance(_model_cfg, str): _hyg_model = _model_cfg elif isinstance(_model_cfg, dict): _hyg_model = _model_cfg.get("default") or _model_cfg.get("model") or _hyg_model # Read explicit context_length override from model config # (same as run_agent.py lines 995-1005) _raw_ctx = _model_cfg.get("context_length") if _raw_ctx is not None: try: _hyg_config_context_length = int(_raw_ctx) except (TypeError, ValueError): pass # Read provider for accurate context detection _hyg_provider = _model_cfg.get("provider") or None _hyg_base_url = _model_cfg.get("base_url") or None # Read compression settings — only use enabled flag. # The threshold is intentionally separate from the agent's # compression.threshold (hygiene runs higher). _comp_cfg = _hyg_data.get("compression", {}) if isinstance(_comp_cfg, dict): _hyg_compression_enabled = str( _comp_cfg.get("enabled", True) ).lower() in ("true", "1", "yes") _raw_hard_limit = _comp_cfg.get("hygiene_hard_message_limit") if _raw_hard_limit is not None: try: _parsed = int(_raw_hard_limit) if _parsed > 0: _hyg_hard_msg_limit = _parsed except (TypeError, ValueError): pass try: _hyg_model, _hyg_runtime = self._resolve_session_agent_runtime( source=source, session_key=session_key, user_config=_hyg_data if isinstance(_hyg_data, dict) else None, ) _hyg_provider = _hyg_runtime.get("provider") or _hyg_provider _hyg_base_url = _hyg_runtime.get("base_url") or _hyg_base_url _hyg_api_key = _hyg_runtime.get("api_key") or _hyg_api_key except Exception: pass # Check custom_providers per-model context_length # (same fallback as run_agent.py lines 1171-1189). # Must run after runtime resolution so _hyg_base_url is set. if _hyg_config_context_length is None and _hyg_base_url: try: try: from hermes_cli.config import get_compatible_custom_providers as _gw_gcp _hyg_custom_providers = _gw_gcp(_hyg_data) except Exception: _hyg_custom_providers = _hyg_data.get("custom_providers") if not isinstance(_hyg_custom_providers, list): _hyg_custom_providers = [] for _cp in _hyg_custom_providers: if not isinstance(_cp, dict): continue _cp_url = (_cp.get("base_url") or "").rstrip("/") if _cp_url and _cp_url == _hyg_base_url.rstrip("/"): _cp_models = _cp.get("models", {}) if isinstance(_cp_models, dict): _cp_model_cfg = _cp_models.get(_hyg_model, {}) if isinstance(_cp_model_cfg, dict): _cp_ctx = _cp_model_cfg.get("context_length") if _cp_ctx is not None: _hyg_config_context_length = int(_cp_ctx) break except (TypeError, ValueError): pass except Exception: pass if _hyg_compression_enabled: _hyg_context_length = get_model_context_length( _hyg_model, base_url=_hyg_base_url or "", api_key=_hyg_api_key or "", config_context_length=_hyg_config_context_length, provider=_hyg_provider or "", ) _compress_token_threshold = int( _hyg_context_length * _hyg_threshold_pct ) _warn_token_threshold = int(_hyg_context_length * 0.95) _msg_count = len(history) # Prefer actual API-reported tokens from the last turn # (stored in session entry) over the rough char-based estimate. _stored_tokens = session_entry.last_prompt_tokens if _stored_tokens > 0: _approx_tokens = _stored_tokens _token_source = "actual" else: _approx_tokens = estimate_messages_tokens_rough(history) _token_source = "estimated" # Note: rough estimates overestimate by 30-50% for code/JSON-heavy # sessions, but that just means hygiene fires a bit early — which # is safe and harmless. The 85% threshold already provides ample # headroom (agent's own compressor runs at 50%). A previous 1.4x # multiplier tried to compensate by inflating the threshold, but # 85% * 1.4 = 119% of context — which exceeds the model's limit # and prevented hygiene from ever firing for ~200K models (GLM-5). # Hard safety valve: force compression if message count is # extreme, regardless of token estimates. This breaks the # death spiral where API disconnects prevent token data # collection, which prevents compression, which causes more # disconnects. 400 messages is well above normal sessions # but catches runaway growth before it becomes unrecoverable. # Threshold is configurable via # compression.hygiene_hard_message_limit. # (#2153) _HARD_MSG_LIMIT = _hyg_hard_msg_limit _needs_compress = ( _approx_tokens >= _compress_token_threshold or _msg_count >= _HARD_MSG_LIMIT ) if _needs_compress: logger.info( "Session hygiene: %s messages, ~%s tokens (%s) — auto-compressing " "(threshold: %s%% of %s = %s tokens)", _msg_count, f"{_approx_tokens:,}", _token_source, int(_hyg_threshold_pct * 100), f"{_hyg_context_length:,}", f"{_compress_token_threshold:,}", ) _hyg_meta = {"thread_id": source.thread_id} if source.thread_id else None try: from run_agent import AIAgent _hyg_model, _hyg_runtime = self._resolve_session_agent_runtime( source=source, session_key=session_key, user_config=_hyg_data if isinstance(_hyg_data, dict) else None, ) if _hyg_runtime.get("api_key"): _hyg_msgs = [ {"role": m.get("role"), "content": m.get("content")} for m in history if m.get("role") in ("user", "assistant") and m.get("content") ] if len(_hyg_msgs) >= 4: _hyg_agent = AIAgent( **_hyg_runtime, model=_hyg_model, max_iterations=4, quiet_mode=True, skip_memory=True, enabled_toolsets=["memory"], session_id=session_entry.session_id, ) try: _hyg_agent._print_fn = lambda *a, **kw: None loop = asyncio.get_running_loop() _compressed, _ = await loop.run_in_executor( None, lambda: _hyg_agent._compress_context( _hyg_msgs, "", approx_tokens=_approx_tokens, ), ) # _compress_context ends the old session and creates # a new session_id. Write compressed messages into # the NEW session so the old transcript stays intact # and searchable via session_search. _hyg_new_sid = _hyg_agent.session_id if _hyg_new_sid != session_entry.session_id: session_entry.session_id = _hyg_new_sid self.session_store._save() self.session_store.rewrite_transcript( session_entry.session_id, _compressed ) # Reset stored token count — transcript was rewritten session_entry.last_prompt_tokens = 0 history = _compressed _new_count = len(_compressed) _new_tokens = estimate_messages_tokens_rough( _compressed ) logger.info( "Session hygiene: compressed %s → %s msgs, " "~%s → ~%s tokens", _msg_count, _new_count, f"{_approx_tokens:,}", f"{_new_tokens:,}", ) if _new_tokens >= _warn_token_threshold: logger.warning( "Session hygiene: still ~%s tokens after " "compression", f"{_new_tokens:,}", ) # If summary generation failed, the # compressor inserted a static fallback # placeholder and the dropped turns are # gone for good. Surface a visible # warning to the gateway user — agent.log # alone is invisible on TG/Discord/etc. _comp = getattr(_hyg_agent, "context_compressor", None) if _comp is not None and getattr(_comp, "_last_summary_fallback_used", False): _dropped = getattr(_comp, "_last_summary_dropped_count", 0) _err = getattr(_comp, "_last_summary_error", None) or "unknown error" _warn_msg = ( "⚠️ Context compression summary failed " f"({_err}). {_dropped} historical message(s) " "were removed and replaced with a placeholder. " "Earlier context is no longer recoverable. " "Consider /reset for a clean session, or check " "your auxiliary.compression model configuration." ) try: _adapter = self.adapters.get(source.platform) if _adapter and source.chat_id: await _adapter.send(source.chat_id, _warn_msg, metadata=_hyg_meta) except Exception as _werr: logger.warning( "Failed to deliver compression-failure warning to user: %s", _werr, ) # Separately: if the user's CONFIGURED aux # model failed and we recovered by falling # back to the main model, tell them — a # misconfigured auxiliary.compression.model # is something only they can fix, and # silent recovery would hide it. elif _comp is not None and getattr(_comp, "_last_aux_model_failure_model", None): _aux_model = getattr(_comp, "_last_aux_model_failure_model", "") _aux_err = getattr(_comp, "_last_aux_model_failure_error", None) or "unknown error" _aux_msg = ( f"ℹ️ Configured compression model `{_aux_model}` " f"failed ({_aux_err}). Recovered using your main " "model — context is intact — but you may want to " "check `auxiliary.compression.model` in config.yaml." ) try: _adapter = self.adapters.get(source.platform) if _adapter and source.chat_id: await _adapter.send(source.chat_id, _aux_msg, metadata=_hyg_meta) except Exception as _werr: logger.warning( "Failed to deliver aux-model-fallback notice to user: %s", _werr, ) finally: self._cleanup_agent_resources(_hyg_agent) except Exception as e: logger.warning( "Session hygiene auto-compress failed: %s", e ) # First-message onboarding -- only on the very first interaction ever if not history and not self.session_store.has_any_sessions(): context_prompt += ( "\n\n[System note: This is the user's very first message ever. " "Briefly introduce yourself and mention that /help shows available commands. " "Keep the introduction concise -- one or two sentences max.]" ) # One-time prompt if no home channel is set for this platform # Skip for webhooks - they deliver directly to configured targets (github_comment, etc.) if not history and source.platform and source.platform != Platform.LOCAL and source.platform != Platform.WEBHOOK: platform_name = source.platform.value env_key = _home_target_env_var(platform_name) if not os.getenv(env_key): # Slack dispatches all Hermes commands through a single # parent slash command `/hermes`; bare `/sethome` is not # registered and would fail with "app did not respond". sethome_cmd = ( "/hermes sethome" if source.platform == Platform.SLACK else "/sethome" ) notice = ( f"📬 No home channel is set for {platform_name.title()}. " f"A home channel is where Hermes delivers cron job results " f"and cross-platform messages.\n\n" f"Type {sethome_cmd} to make this chat your home channel, " f"or ignore to skip." ) await self._deliver_platform_notice(source, notice) # ----------------------------------------------------------------- # Voice channel awareness — inject current voice channel state # into context so the agent knows who is in the channel and who # is speaking, without needing a separate tool call. # ----------------------------------------------------------------- if source.platform == Platform.DISCORD: adapter = self.adapters.get(Platform.DISCORD) guild_id = self._get_guild_id(event) if guild_id and adapter and hasattr(adapter, "get_voice_channel_context"): vc_context = adapter.get_voice_channel_context(guild_id) if vc_context: context_prompt += f"\n\n{vc_context}" # ----------------------------------------------------------------- # Auto-analyze images sent by the user # # If the user attached image(s), we run the vision tool eagerly so # the conversation model always receives a text description. The # local file path is also included so the model can re-examine the # image later with a more targeted question via vision_analyze. # # We filter to image paths only (by media_type) so that non-image # attachments (documents, audio, etc.) are not sent to the vision # tool even when they appear in the same message. # ----------------------------------------------------------------- message_text = await self._prepare_inbound_message_text( event=event, source=source, history=history, ) if message_text is None: return # Bind this gateway run generation to the adapter's active-session # event so deferred post-delivery callbacks can be released by the # same run that registered them. self._bind_adapter_run_generation( self.adapters.get(source.platform), session_key, run_generation, ) try: # Emit agent:start hook hook_ctx = { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "session_id": session_entry.session_id, "message": message_text[:500], } await self.hooks.emit("agent:start", hook_ctx) # Run the agent agent_result = await self._run_agent( message=message_text, context_prompt=context_prompt, history=history, source=source, session_id=session_entry.session_id, session_key=session_key, run_generation=run_generation, event_message_id=event.message_id, channel_prompt=event.channel_prompt, ) # Stop persistent typing indicator now that the agent is done try: _typing_adapter = self.adapters.get(source.platform) if _typing_adapter and hasattr(_typing_adapter, "stop_typing"): await _typing_adapter.stop_typing(source.chat_id) except Exception: pass if not self._is_session_run_current(_quick_key, run_generation): logger.info( "Discarding stale agent result for %s — generation %d is no longer current", _quick_key or "?", run_generation, ) _stale_adapter = self.adapters.get(source.platform) if getattr(type(_stale_adapter), "pop_post_delivery_callback", None) is not None: _stale_adapter.pop_post_delivery_callback( _quick_key, generation=run_generation, ) elif _stale_adapter and hasattr(_stale_adapter, "_post_delivery_callbacks"): _stale_adapter._post_delivery_callbacks.pop(_quick_key, None) return None response = agent_result.get("final_response") or "" # Convert the agent's internal "(empty)" sentinel into a # user-friendly message. "(empty)" means the model failed to # produce visible content after exhausting all retries (nudge, # prefill, empty-retry, fallback). Sending the raw sentinel # looks like a bug; a short explanation is more helpful. if response == "(empty)": response = ( "⚠️ The model returned no response after processing tool " "results. This can happen with some models — try again or " "rephrase your question." ) agent_messages = agent_result.get("messages", []) _response_time = time.time() - _msg_start_time _api_calls = agent_result.get("api_calls", 0) _resp_len = len(response) logger.info( "response ready: platform=%s chat=%s time=%.1fs api_calls=%d response=%d chars", _platform_name, source.chat_id or "unknown", _response_time, _api_calls, _resp_len, ) # Successful turn — clear any stuck-loop counter for this session. # This ensures the counter only accumulates across CONSECUTIVE # restarts where the session was active (never completed). # # Also clear the resume_pending flag (set by drain-timeout # shutdown) — the turn ran to completion, so recovery # succeeded and subsequent messages should no longer receive # the restart-interruption system note. if session_key: self._clear_restart_failure_count(session_key) try: self.session_store.clear_resume_pending(session_key) except Exception as _e: logger.debug( "clear_resume_pending failed for %s: %s", session_key, _e, ) # Surface error details when the agent failed silently (final_response=None) if not response and agent_result.get("failed"): error_detail = agent_result.get("error", "unknown error") error_str = str(error_detail).lower() # Detect context-overflow failures and give specific guidance. # Generic 400 "Error" from Anthropic with large sessions is the # most common cause of this (#1630). _is_ctx_fail = any(p in error_str for p in ( "context", "token", "too large", "too long", "exceed", "payload", )) or ( "400" in error_str and len(history) > 50 ) if _is_ctx_fail: response = ( "⚠️ Session too large for the model's context window.\n" "Use /compact to compress the conversation, or " "/reset to start fresh." ) else: response = ( f"The request failed: {str(error_detail)[:300]}\n" "Try again or use /reset to start a fresh session." ) # If the agent's session_id changed during compression, update # session_entry so transcript writes below go to the right session. if agent_result.get("session_id") and agent_result["session_id"] != session_entry.session_id: session_entry.session_id = agent_result["session_id"] # Prepend reasoning/thinking if display is enabled (per-platform) try: from gateway.display_config import resolve_display_setting as _rds _show_reasoning_effective = _rds( _load_gateway_config(), _platform_config_key(source.platform), "show_reasoning", getattr(self, "_show_reasoning", False), ) except Exception: _show_reasoning_effective = getattr(self, "_show_reasoning", False) if _show_reasoning_effective and response: last_reasoning = agent_result.get("last_reasoning") if last_reasoning: # Collapse long reasoning to keep messages readable lines = last_reasoning.strip().splitlines() if len(lines) > 15: display_reasoning = "\n".join(lines[:15]) display_reasoning += f"\n_... ({len(lines) - 15} more lines)_" else: display_reasoning = last_reasoning.strip() response = f"💭 **Reasoning:**\n```\n{display_reasoning}\n```\n\n{response}" # Runtime-metadata footer — only on the FINAL message of the turn. # Off by default (display.runtime_footer.enabled=false). When # streaming already delivered the body, we can't mutate the sent # text, so we fire a separate trailing send below. _footer_line = "" try: from gateway.runtime_footer import build_footer_line as _bfl _footer_line = _bfl( user_config=_load_gateway_config(), platform_key=_platform_config_key(source.platform), model=agent_result.get("model"), context_tokens=agent_result.get("last_prompt_tokens", 0) or 0, context_length=agent_result.get("context_length") or None, cwd=os.environ.get("TERMINAL_CWD", ""), ) except Exception as _footer_err: logger.debug("runtime_footer build failed: %s", _footer_err) _footer_line = "" if _footer_line and response and not agent_result.get("already_sent"): response = f"{response}\n\n{_footer_line}" # Emit agent:end hook await self.hooks.emit("agent:end", { **hook_ctx, "response": (response or "")[:500], }) # Check for pending process watchers (check_interval on background processes) try: from tools.process_registry import process_registry while process_registry.pending_watchers: watcher = process_registry.pending_watchers.pop(0) asyncio.create_task(self._run_process_watcher(watcher)) except Exception as e: logger.error("Process watcher setup error: %s", e) # Drain watch pattern notifications that arrived during the agent run. # Watch events and completions share the same queue; completions are # already handled by the per-process watcher task above, so we only # inject watch-type events here. try: from tools.process_registry import process_registry as _pr _watch_events = [] while not _pr.completion_queue.empty(): evt = _pr.completion_queue.get_nowait() evt_type = evt.get("type", "completion") if evt_type in ("watch_match", "watch_disabled"): _watch_events.append(evt) # else: completion events are handled by the watcher task for evt in _watch_events: synth_text = _format_gateway_process_notification(evt) if synth_text: try: await self._inject_watch_notification(synth_text, evt) except Exception as e2: logger.error("Watch notification injection error: %s", e2) except Exception as e: logger.debug("Watch queue drain error: %s", e) # NOTE: Dangerous command approvals are now handled inline by the # blocking gateway approval mechanism in tools/approval.py. The agent # thread blocks until the user responds with /approve or /deny, so by # the time we reach here the approval has already been resolved. The # old post-loop pop_pending + approval_hint code was removed in favour # of the blocking approach that mirrors CLI's synchronous input(). # Save the full conversation to the transcript, including tool calls. # This preserves the complete agent loop (tool_calls, tool results, # intermediate reasoning) so sessions can be resumed with full context # and transcripts are useful for debugging and training data. # # IMPORTANT: For context-overflow failures (compression exhausted, # generic 400 on large sessions) we must NOT persist the user's # message — doing so would grow the session further and cause the # same failure on the next attempt, an infinite loop. (#1630, #9893) # # Transient failures (429, timeout, connection error, provider 5xx) # are different: the session is not oversized, and silently dropping # the user message causes severe context loss on retry — the agent # forgets what was just asked. Persist the user turn so the # conversation is preserved. (#7100) agent_failed_early = bool(agent_result.get("failed")) _err_str_for_classify = str(agent_result.get("error", "")).lower() # Use specific multi-word phrases (not bare "exceed" or "token") # to avoid false positives on transient errors like "rate limit # exceeded" or "invalid auth token". Matches run_agent.py's # own context-length classifier. is_context_overflow_failure = agent_failed_early and ( bool(agent_result.get("compression_exhausted")) or any(p in _err_str_for_classify for p in ( "context length", "context size", "context window", "maximum context", "token limit", "too many tokens", "reduce the length", "exceeds the limit", "request entity too large", "prompt is too long", "payload too large", "input is too long", )) or ("400" in _err_str_for_classify and len(history) > 50) ) if is_context_overflow_failure: logger.info( "Skipping transcript persistence for context-overflow " "failure in session %s to prevent session growth loop.", session_entry.session_id, ) elif agent_failed_early: logger.info( "Transient agent failure in session %s — persisting user " "message so conversation context is preserved on retry.", session_entry.session_id, ) # When compression is exhausted, the session is permanently too # large to process. Auto-reset it so the next message starts # fresh instead of replaying the same oversized context in an # infinite fail loop. (#9893) if agent_result.get("compression_exhausted") and session_entry and session_key: logger.info( "Auto-resetting session %s after compression exhaustion.", session_entry.session_id, ) self.session_store.reset_session(session_key) self._evict_cached_agent(session_key) self._session_model_overrides.pop(session_key, None) self._set_session_reasoning_override(session_key, None) if hasattr(self, "_pending_model_notes"): self._pending_model_notes.pop(session_key, None) response = (response or "") + ( "\n\n🔄 Session auto-reset — the conversation exceeded the " "maximum context size and could not be compressed further. " "Your next message will start a fresh session." ) ts = datetime.now().isoformat() # If this is a fresh session (no history), write the full tool # definitions as the first entry so the transcript is self-describing # -- the same list of dicts sent as tools=[...] in the API request. if is_context_overflow_failure: pass # Skip all transcript writes — don't grow a broken session elif not history: tool_defs = agent_result.get("tools", []) self.session_store.append_to_transcript( session_entry.session_id, { "role": "session_meta", "tools": tool_defs or [], "model": _resolve_gateway_model(), "platform": source.platform.value if source.platform else "", "timestamp": ts, } ) # Find only the NEW messages from this turn (skip history we loaded). # Use the filtered history length (history_offset) that was actually # passed to the agent, not len(history) which includes session_meta # entries that were stripped before the agent saw them. if is_context_overflow_failure: pass # handled above — skip all transcript writes elif agent_failed_early: # Transient failure (429/timeout/5xx): persist only the user # message so the next message can load a transcript that # reflects what was said. Skip the assistant error text since # it's a gateway-generated hint, not model output. (#7100) self.session_store.append_to_transcript( session_entry.session_id, {"role": "user", "content": message_text, "timestamp": ts}, ) else: history_len = agent_result.get("history_offset", len(history)) new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else [] # If no new messages found (edge case), fall back to simple user/assistant if not new_messages: self.session_store.append_to_transcript( session_entry.session_id, {"role": "user", "content": message_text, "timestamp": ts} ) if response: self.session_store.append_to_transcript( session_entry.session_id, {"role": "assistant", "content": response, "timestamp": ts} ) else: # The agent already persisted these messages to SQLite via # _flush_messages_to_session_db(), so skip the DB write here # to prevent the duplicate-write bug (#860). We still write # to JSONL for backward compatibility and as a backup. agent_persisted = self._session_db is not None for msg in new_messages: # Skip system messages (they're rebuilt each run) if msg.get("role") == "system": continue # Add timestamp to each message for debugging entry = {**msg, "timestamp": ts} self.session_store.append_to_transcript( session_entry.session_id, entry, skip_db=agent_persisted, ) # Token counts and model are now persisted by the agent directly. # Keep only last_prompt_tokens here for context-window tracking and # compression decisions. self.session_store.update_session( session_entry.session_key, last_prompt_tokens=agent_result.get("last_prompt_tokens", 0), ) # Auto voice reply: send TTS audio before the text response _already_sent = bool(agent_result.get("already_sent")) if self._should_send_voice_reply(event, response, agent_messages, already_sent=_already_sent): await self._send_voice_reply(event, response) # If streaming already delivered the response, extract and # deliver any MEDIA: files before returning None. Streaming # sends raw text chunks that include MEDIA: tags — the normal # post-processing in _process_message_background is skipped # when already_sent is True, so media files would never be # delivered without this. # # Never skip when the agent failed — the error message is new # content the user hasn't seen (streaming only sent earlier # partial output before the failure). Without this guard, # users see the agent "stop responding without explanation." if agent_result.get("already_sent") and not agent_result.get("failed"): if response: _media_adapter = self.adapters.get(source.platform) if _media_adapter: await self._deliver_media_from_response( response, event, _media_adapter, ) # Streaming already delivered the body text, but the footer was # intentionally held back (see the `not already_sent` gate above). # Send it now as a small trailing message so Telegram/Discord/etc. # still surface the runtime metadata on the final reply. if _footer_line: try: _foot_adapter = self.adapters.get(source.platform) if _foot_adapter: await _foot_adapter.send(source.chat_id, _footer_line) except Exception as _e: logger.debug("trailing footer send failed: %s", _e) return None return response except Exception as e: # Stop typing indicator on error too try: _err_adapter = self.adapters.get(source.platform) if _err_adapter and hasattr(_err_adapter, "stop_typing"): await _err_adapter.stop_typing(source.chat_id) except Exception: pass logger.exception("Agent error in session %s", session_key) error_type = type(e).__name__ error_detail = str(e)[:300] if str(e) else "no details available" status_hint = "" status_code = getattr(e, "status_code", None) _hist_len = len(history) if 'history' in locals() else 0 if status_code == 401: status_hint = " Check your API key or run `claude /login` to refresh OAuth credentials." elif status_code == 402: status_hint = " Your API balance or quota is exhausted. Check your provider dashboard." elif status_code == 429: # Check if this is a plan usage limit (resets on a schedule) vs a transient rate limit _err_body = getattr(e, "response", None) _err_json = {} try: if _err_body is not None: _err_json = _err_body.json().get("error", {}) except Exception: pass if _err_json.get("type") == "usage_limit_reached": _resets_in = _err_json.get("resets_in_seconds") if _resets_in and _resets_in > 0: import math _hours = math.ceil(_resets_in / 3600) status_hint = f" Your plan's usage limit has been reached. It resets in ~{_hours}h." else: status_hint = " Your plan's usage limit has been reached. Please wait until it resets." else: status_hint = " You are being rate-limited. Please wait a moment and try again." elif status_code == 529: status_hint = " The API is temporarily overloaded. Please try again shortly." elif status_code in (400, 500): # 400 with a large session is context overflow. # 500 with a large session often means the payload is too large # for the API to process — treat it the same way. if _hist_len > 50: return ( "⚠️ Session too large for the model's context window.\n" "Use /compact to compress the conversation, or " "/reset to start fresh." ) elif status_code == 400: status_hint = " The request was rejected by the API." return ( f"Sorry, I encountered an error ({error_type}).\n" f"{error_detail}\n" f"{status_hint}" "Try again or use /reset to start a fresh session." ) finally: # Restore session context variables to their pre-handler state self._clear_session_env(_session_env_tokens) def _format_session_info(self) -> str: """Resolve current model config and return a formatted info block. Surfaces model, provider, context length, and endpoint so gateway users can immediately see if context detection went wrong (e.g. local models falling to the 128K default). """ from agent.model_metadata import get_model_context_length, DEFAULT_FALLBACK_CONTEXT model = _resolve_gateway_model() config_context_length = None provider = None base_url = None api_key = None custom_provs = None try: data = _load_gateway_config() if data: model_cfg = data.get("model", {}) if isinstance(model_cfg, dict): raw_ctx = model_cfg.get("context_length") if raw_ctx is not None: try: config_context_length = int(raw_ctx) except (TypeError, ValueError): pass provider = model_cfg.get("provider") or None base_url = model_cfg.get("base_url") or None try: from hermes_cli.config import get_compatible_custom_providers custom_provs = get_compatible_custom_providers(data) except Exception: custom_provs = data.get("custom_providers") except Exception: pass # Resolve runtime credentials for probing try: runtime = _resolve_runtime_agent_kwargs() provider = provider or runtime.get("provider") base_url = base_url or runtime.get("base_url") api_key = runtime.get("api_key") except Exception: pass context_length = get_model_context_length( model, base_url=base_url or "", api_key=api_key or "", config_context_length=config_context_length, provider=provider or "", custom_providers=custom_provs, ) # Format context source hint if config_context_length is not None: ctx_source = "config" elif context_length == DEFAULT_FALLBACK_CONTEXT: ctx_source = "default — set model.context_length in config to override" else: ctx_source = "detected" # Format context length for display if context_length >= 1_000_000: ctx_display = f"{context_length / 1_000_000:.1f}M" elif context_length >= 1_000: ctx_display = f"{context_length // 1_000}K" else: ctx_display = str(context_length) lines = [ f"◆ Model: `{model}`", f"◆ Provider: {provider or 'openrouter'}", f"◆ Context: {ctx_display} tokens ({ctx_source})", ] # Show endpoint for local/custom setups if base_url and ("localhost" in base_url or "127.0.0.1" in base_url or "0.0.0.0" in base_url): lines.append(f"◆ Endpoint: {base_url}") return "\n".join(lines) async def _handle_reset_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: """Handle /new or /reset command.""" source = event.source # Get existing session key session_key = self._session_key_for_source(source) self._invalidate_session_run_generation(session_key, reason="session_reset") # Snapshot the old entry so on_session_finalize can report the # expiring session id before reset_session() rotates it. old_entry = self.session_store._entries.get(session_key) # Close tool resources on the old agent (terminal sandboxes, browser # daemons, background processes) before evicting from cache. # Guard with getattr because test fixtures may skip __init__. _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is not None: with _cache_lock: _cached = self._agent_cache.get(session_key) _old_agent = _cached[0] if isinstance(_cached, tuple) else _cached if _cached else None if _old_agent is not None: self._cleanup_agent_resources(_old_agent) self._evict_cached_agent(session_key) # Discard any /queue overflow for this session — /new is a # conversation-boundary operation, queued follow-ups from the # previous conversation must not bleed into the new one. _qe = getattr(self, "_queued_events", None) if _qe is not None: _qe.pop(session_key, None) try: from tools.env_passthrough import clear_env_passthrough clear_env_passthrough() except Exception: pass try: from tools.credential_files import clear_credential_files clear_credential_files() except Exception: pass # Reset the session new_entry = self.session_store.reset_session(session_key) # Clear any session-scoped model/reasoning overrides so the next agent # picks up configured defaults instead of previous session switches. self._session_model_overrides.pop(session_key, None) self._set_session_reasoning_override(session_key, None) if hasattr(self, "_pending_model_notes"): self._pending_model_notes.pop(session_key, None) # Clear session-scoped dangerous-command approvals and /yolo state. # /new is a conversation-boundary operation — approval state from the # previous conversation must not survive the reset. self._clear_session_boundary_security_state(session_key) # Fire plugin on_session_finalize hook (session boundary) try: from hermes_cli.plugins import invoke_hook as _invoke_hook _old_sid = old_entry.session_id if old_entry else None _invoke_hook("on_session_finalize", session_id=_old_sid, platform=source.platform.value if source.platform else "") except Exception: pass # Emit session:end hook (session is ending) await self.hooks.emit("session:end", { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "session_key": session_key, }) # Emit session:reset hook await self.hooks.emit("session:reset", { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "session_key": session_key, }) # Resolve session config info to surface to the user try: session_info = self._format_session_info() except Exception: session_info = "" if new_entry: header = "✨ Session reset! Starting fresh." else: # No existing session, just create one new_entry = self.session_store.get_or_create_session(source, force_new=True) header = "✨ New session started!" # Fire plugin on_session_reset hook (new session guaranteed to exist) try: from hermes_cli.plugins import invoke_hook as _invoke_hook _new_sid = new_entry.session_id if new_entry else None _invoke_hook("on_session_reset", session_id=_new_sid, platform=source.platform.value if source.platform else "") except Exception: pass # Append a random tip to the reset message try: from hermes_cli.tips import get_random_tip _tip_line = f"\n✦ Tip: {get_random_tip()}" except Exception: _tip_line = "" if session_info: return EphemeralReply(f"{header}\n\n{session_info}{_tip_line}") return EphemeralReply(f"{header}{_tip_line}") async def _handle_profile_command(self, event: MessageEvent) -> str: """Handle /profile — show active profile name and home directory.""" from hermes_constants import display_hermes_home from hermes_cli.profiles import get_active_profile_name display = display_hermes_home() profile_name = get_active_profile_name() lines = [ f"👤 **Profile:** `{profile_name}`", f"📂 **Home:** `{display}`", ] return "\n".join(lines) async def _handle_kanban_command(self, event: MessageEvent) -> str: """Handle /kanban — delegate to the shared kanban CLI. Run the potentially-blocking DB work in a thread pool so the gateway event loop stays responsive. Read operations (list, show, context, tail) are permitted while an agent is running; mutations are allowed too because the board is profile-agnostic and does not touch the running agent's state. For ``/kanban create`` invocations we also auto-subscribe the originating gateway source (platform + chat + thread) to the new task's terminal events, so the user hears back when the worker completes / blocks / auto-blocks / crashes without having to poll. """ import asyncio import re from hermes_cli.kanban import run_slash text = (event.text or "").strip() # Strip the leading "/kanban" (with or without slash), leaving args. if text.startswith("/"): text = text.lstrip("/") if text.startswith("kanban"): text = text[len("kanban"):].lstrip() is_create = text.split(None, 1)[:1] == ["create"] try: output = await asyncio.to_thread(run_slash, text) except Exception as exc: # pragma: no cover - defensive return f"⚠ kanban error: {exc}" # Auto-subscribe on create. Parse the task id from the CLI's standard # success line ("Created t_abcd (ready, assignee=...)"). If the user # passed --json we don't subscribe; they're clearly scripting and # can call /kanban notify-subscribe explicitly. if is_create and output: m = re.search(r"Created\s+(t_[0-9a-f]+)\b", output) if m: task_id = m.group(1) try: source = event.source platform = getattr(source, "platform", None) platform_str = ( platform.value if hasattr(platform, "value") else str(platform or "") ).lower() chat_id = str(getattr(source, "chat_id", "") or "") thread_id = str(getattr(source, "thread_id", "") or "") user_id = str(getattr(source, "user_id", "") or "") or None if platform_str and chat_id: def _sub(): from hermes_cli import kanban_db as _kb conn = _kb.connect() try: _kb.add_notify_sub( conn, task_id=task_id, platform=platform_str, chat_id=chat_id, thread_id=thread_id or None, user_id=user_id, ) finally: conn.close() await asyncio.to_thread(_sub) output = ( output.rstrip() + f"\n(subscribed — you'll be notified when {task_id} " f"completes or blocks)" ) except Exception as exc: logger.warning("kanban create auto-subscribe failed: %s", exc) # Gateway messages have practical length caps; truncate long # listings to keep the UX reasonable. if len(output) > 3800: output = output[:3800] + "\n… (truncated; use `hermes kanban …` in your terminal for full output)" return output or "(no output)" async def _handle_status_command(self, event: MessageEvent) -> str: """Handle /status command.""" source = event.source session_entry = self.session_store.get_or_create_session(source) connected_platforms = [p.value for p in self.adapters.keys()] # Check if there's an active agent session_key = session_entry.session_key is_running = session_key in self._running_agents # Count pending /queue follow-ups (slot + overflow). adapter = self.adapters.get(source.platform) if source else None queue_depth = self._queue_depth(session_key, adapter=adapter) title = None # Pull token totals from the SQLite session DB rather than the # in-memory SessionStore. The agent's per-turn token deltas are # persisted into sessions_db (run_agent.py), not into SessionEntry, # so session_entry.total_tokens is always 0. SessionDB is the # single source of truth; reading it here keeps /status accurate # without duplicating token writes into two stores. db_total_tokens = 0 if self._session_db: try: title = self._session_db.get_session_title(session_entry.session_id) except Exception: title = None try: row = self._session_db.get_session(session_entry.session_id) if row: db_total_tokens = ( (row.get("input_tokens") or 0) + (row.get("output_tokens") or 0) + (row.get("cache_read_tokens") or 0) + (row.get("cache_write_tokens") or 0) + (row.get("reasoning_tokens") or 0) ) except Exception: db_total_tokens = 0 lines = [ "📊 **Hermes Gateway Status**", "", f"**Session ID:** `{session_entry.session_id}`", ] if title: lines.append(f"**Title:** {title}") lines.extend([ f"**Created:** {session_entry.created_at.strftime('%Y-%m-%d %H:%M')}", f"**Last Activity:** {session_entry.updated_at.strftime('%Y-%m-%d %H:%M')}", f"**Tokens:** {db_total_tokens:,}", f"**Agent Running:** {'Yes ⚡' if is_running else 'No'}", ]) if queue_depth: lines.append(f"**Queued follow-ups:** {queue_depth}") lines.extend([ "", f"**Connected Platforms:** {', '.join(connected_platforms)}", ]) return "\n".join(lines) async def _handle_agents_command(self, event: MessageEvent) -> str: """Handle /agents command - list active agents and running tasks.""" from tools.process_registry import format_uptime_short, process_registry now = time.time() current_session_key = self._session_key_for_source(event.source) running_agents: dict = getattr(self, "_running_agents", {}) or {} running_started: dict = getattr(self, "_running_agents_ts", {}) or {} agent_rows: list[dict] = [] for session_key, agent in running_agents.items(): started = float(running_started.get(session_key, now)) elapsed = max(0, int(now - started)) is_pending = agent is _AGENT_PENDING_SENTINEL agent_rows.append( { "session_key": session_key, "elapsed": elapsed, "state": "starting" if is_pending else "running", "session_id": "" if is_pending else str(getattr(agent, "session_id", "") or ""), "model": "" if is_pending else str(getattr(agent, "model", "") or ""), } ) agent_rows.sort(key=lambda row: row["elapsed"], reverse=True) running_processes: list[dict] = [] try: running_processes = [ p for p in process_registry.list_sessions() if p.get("status") == "running" ] except Exception: running_processes = [] background_tasks = [ t for t in (getattr(self, "_background_tasks", set()) or set()) if hasattr(t, "done") and not t.done() ] lines = [ "🤖 **Active Agents & Tasks**", "", f"**Active agents:** {len(agent_rows)}", ] if agent_rows: for idx, row in enumerate(agent_rows[:12], 1): current = " · this chat" if row["session_key"] == current_session_key else "" sid = f" · `{row['session_id']}`" if row["session_id"] else "" model = f" · `{row['model']}`" if row["model"] else "" lines.append( f"{idx}. `{row['session_key']}` · {row['state']} · " f"{format_uptime_short(row['elapsed'])}{sid}{model}{current}" ) if len(agent_rows) > 12: lines.append(f"... and {len(agent_rows) - 12} more") lines.extend( [ "", f"**Running background processes:** {len(running_processes)}", ] ) if running_processes: for proc in running_processes[:12]: cmd = " ".join(str(proc.get("command", "")).split()) if len(cmd) > 90: cmd = cmd[:87] + "..." lines.append( f"- `{proc.get('session_id', '?')}` · " f"{format_uptime_short(int(proc.get('uptime_seconds', 0)))} · `{cmd}`" ) if len(running_processes) > 12: lines.append(f"... and {len(running_processes) - 12} more") lines.extend( [ "", f"**Gateway async jobs:** {len(background_tasks)}", ] ) if not agent_rows and not running_processes and not background_tasks: lines.append("") lines.append("No active agents or running tasks.") return "\n".join(lines) async def _handle_stop_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: """Handle /stop command - interrupt a running agent. When an agent is truly hung (blocked thread that never checks _interrupt_requested), the early intercept in _handle_message() handles /stop before this method is reached. This handler fires only through normal command dispatch (no running agent) or as a fallback. Force-clean the session lock in all cases for safety. The session is preserved so the user can continue the conversation. """ source = event.source session_entry = self.session_store.get_or_create_session(source) session_key = session_entry.session_key agent = self._running_agents.get(session_key) if agent is _AGENT_PENDING_SENTINEL: # Force-clean the sentinel so the session is unlocked. await self._interrupt_and_clear_session( session_key, source, interrupt_reason=_INTERRUPT_REASON_STOP, invalidation_reason="stop_command_pending", ) logger.info("STOP (pending) for session %s — sentinel cleared", session_key) return EphemeralReply("⚡ Stopped. The agent hadn't started yet — you can continue this session.") if agent: # Force-clean the session lock so a truly hung agent doesn't # keep it locked forever. await self._interrupt_and_clear_session( session_key, source, interrupt_reason=_INTERRUPT_REASON_STOP, invalidation_reason="stop_command_handler", ) return EphemeralReply("⚡ Stopped. You can continue this session.") else: return "No active task to stop." async def _handle_restart_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: """Handle /restart command - drain active work, then restart the gateway.""" # Defensive idempotency check: if the previous gateway process # recorded this same /restart (same platform + update_id) and the new # process is seeing it *again*, this is a re-delivery caused by PTB's # graceful-shutdown `get_updates` ACK failing on the way out ("Error # while calling `get_updates` one more time to mark all fetched # updates. Suppressing error to ensure graceful shutdown. When # polling for updates is restarted, updates may be received twice." # in gateway.log). Ignoring the stale redelivery prevents a # self-perpetuating restart loop where every fresh gateway # re-processes the same /restart command and immediately restarts # again. if self._is_stale_restart_redelivery(event): logger.info( "Ignoring redelivered /restart (platform=%s, update_id=%s) — " "already processed by a previous gateway instance.", event.source.platform.value if event.source and event.source.platform else "?", event.platform_update_id, ) return "" if self._restart_requested or self._draining: count = self._running_agent_count() if count: return f"⏳ Draining {count} active agent(s) before restart..." return EphemeralReply("⏳ Gateway restart already in progress...") # Save the requester's routing info so the new gateway process can # notify them once it comes back online. try: notify_data = { "platform": event.source.platform.value if event.source.platform else None, "chat_id": event.source.chat_id, } if event.source.thread_id: notify_data["thread_id"] = event.source.thread_id atomic_json_write( _hermes_home / ".restart_notify.json", notify_data, indent=None, ) except Exception as e: logger.debug("Failed to write restart notify file: %s", e) # Record the triggering platform + update_id in a dedicated dedup # marker. Unlike .restart_notify.json (which gets unlinked once the # new gateway sends the "gateway restarted" notification), this # marker persists so the new gateway can still detect a delayed # /restart redelivery from Telegram. Overwritten on every /restart. try: dedup_data = { "platform": event.source.platform.value if event.source.platform else None, "requested_at": time.time(), } if event.platform_update_id is not None: dedup_data["update_id"] = event.platform_update_id atomic_json_write( _hermes_home / ".restart_last_processed.json", dedup_data, indent=None, ) except Exception as e: logger.debug("Failed to write restart dedup marker: %s", e) active_agents = self._running_agent_count() # When running under a service manager (systemd/launchd), use the # service restart path: exit with code 75 so the service manager # restarts us. The detached subprocess approach (setsid + bash) # doesn't work under systemd because KillMode=mixed kills all # processes in the cgroup, including the detached helper. _under_service = bool(os.environ.get("INVOCATION_ID")) # systemd sets this if _under_service: self.request_restart(detached=False, via_service=True) else: self.request_restart(detached=True, via_service=False) if active_agents: return f"⏳ Draining {active_agents} active agent(s) before restart..." return EphemeralReply("♻ Restarting gateway. If you aren't notified within 60 seconds, restart from the console with `hermes gateway restart`.") def _is_stale_restart_redelivery(self, event: MessageEvent) -> bool: """Return True if this /restart is a Telegram re-delivery we already handled. The previous gateway wrote ``.restart_last_processed.json`` with the triggering platform + update_id when it processed the /restart. If we now see a /restart on the same platform with an update_id <= that recorded value AND the marker is recent (< 5 minutes), it's a redelivery and should be ignored. Only applies to Telegram today (the only platform that exposes a numeric cross-session update ordering); other platforms return False. """ if event is None or event.source is None: return False if event.platform_update_id is None: return False if event.source.platform is None: return False # Only Telegram populates platform_update_id currently; be explicit # so future platforms aren't accidentally gated by this check. try: platform_value = event.source.platform.value except Exception: return False if platform_value != "telegram": return False try: marker_path = _hermes_home / ".restart_last_processed.json" if not marker_path.exists(): return False data = json.loads(marker_path.read_text()) except Exception: return False if data.get("platform") != platform_value: return False recorded_uid = data.get("update_id") if not isinstance(recorded_uid, int): return False # Staleness guard: ignore markers older than 5 minutes. A legitimately # old marker (e.g. crash recovery where notify never fired) should not # swallow a fresh /restart from the user. requested_at = data.get("requested_at") if isinstance(requested_at, (int, float)): if time.time() - requested_at > 300: return False return event.platform_update_id <= recorded_uid async def _handle_help_command(self, event: MessageEvent) -> str: """Handle /help command - list available commands.""" from hermes_cli.commands import gateway_help_lines lines = [ "📖 **Hermes Commands**\n", *gateway_help_lines(), ] try: from agent.skill_commands import get_skill_commands skill_cmds = get_skill_commands() if skill_cmds: lines.append(f"\n⚡ **Skill Commands** ({len(skill_cmds)} active):") # Show first 10, then point to /commands for the rest sorted_cmds = sorted(skill_cmds) for cmd in sorted_cmds[:10]: lines.append(f"`{cmd}` — {skill_cmds[cmd]['description']}") if len(sorted_cmds) > 10: lines.append(f"\n... and {len(sorted_cmds) - 10} more. Use `/commands` for the full paginated list.") except Exception: pass return "\n".join(lines) async def _handle_commands_command(self, event: MessageEvent) -> str: """Handle /commands [page] - paginated list of all commands and skills.""" from hermes_cli.commands import gateway_help_lines raw_args = event.get_command_args().strip() if raw_args: try: requested_page = int(raw_args) except ValueError: return "Usage: `/commands [page]`" else: requested_page = 1 # Build combined entry list: built-in commands + skill commands entries = list(gateway_help_lines()) try: from agent.skill_commands import get_skill_commands skill_cmds = get_skill_commands() if skill_cmds: entries.append("") entries.append("⚡ **Skill Commands**:") for cmd in sorted(skill_cmds): desc = skill_cmds[cmd].get("description", "").strip() or "Skill command" entries.append(f"`{cmd}` — {desc}") except Exception: pass if not entries: return "No commands available." from gateway.config import Platform page_size = 15 if event.source.platform == Platform.TELEGRAM else 20 total_pages = max(1, (len(entries) + page_size - 1) // page_size) page = max(1, min(requested_page, total_pages)) start = (page - 1) * page_size page_entries = entries[start:start + page_size] lines = [ f"📚 **Commands** ({len(entries)} total, page {page}/{total_pages})", "", *page_entries, ] if total_pages > 1: nav_parts = [] if page > 1: nav_parts.append(f"`/commands {page - 1}` ← prev") if page < total_pages: nav_parts.append(f"next → `/commands {page + 1}`") lines.extend(["", " | ".join(nav_parts)]) if page != requested_page: lines.append(f"_(Requested page {requested_page} was out of range, showing page {page}.)_") return "\n".join(lines) async def _handle_model_command(self, event: MessageEvent) -> Optional[str]: """Handle /model command — switch model for this session. Supports: /model — interactive picker (Telegram/Discord) or text list /model — switch for this session only /model --global — switch and persist to config.yaml /model --provider — switch provider + model /model --provider — switch to provider, auto-detect model """ import yaml from hermes_cli.model_switch import ( switch_model as _switch_model, parse_model_flags, list_authenticated_providers, ) from hermes_cli.providers import get_label raw_args = event.get_command_args().strip() # Parse --provider and --global flags model_input, explicit_provider, persist_global = parse_model_flags(raw_args) # Read current model/provider from config current_model = "" current_provider = "openrouter" current_base_url = "" current_api_key = "" user_provs = None custom_provs = None config_path = _hermes_home / "config.yaml" try: cfg = _load_gateway_config() if cfg: model_cfg = cfg.get("model", {}) if isinstance(model_cfg, dict): current_model = model_cfg.get("default", "") current_provider = model_cfg.get("provider", current_provider) current_base_url = model_cfg.get("base_url", "") user_provs = cfg.get("providers") try: from hermes_cli.config import get_compatible_custom_providers custom_provs = get_compatible_custom_providers(cfg) except Exception: custom_provs = cfg.get("custom_providers") except Exception: pass # Check for session override source = event.source session_key = self._session_key_for_source(source) override = self._session_model_overrides.get(session_key, {}) if override: current_model = override.get("model", current_model) current_provider = override.get("provider", current_provider) current_base_url = override.get("base_url", current_base_url) current_api_key = override.get("api_key", current_api_key) # No args: show interactive picker (Telegram/Discord) or text list if not model_input and not explicit_provider: # Try interactive picker if the platform supports it adapter = self.adapters.get(source.platform) has_picker = ( adapter is not None and getattr(type(adapter), "send_model_picker", None) is not None ) if has_picker: try: providers = list_authenticated_providers( current_provider=current_provider, current_base_url=current_base_url, current_model=current_model, user_providers=user_provs, custom_providers=custom_provs, max_models=50, ) except Exception: providers = [] if providers: # Build a callback closure for when the user picks a model. # Captures self + locals needed for the switch logic. _self = self _session_key = session_key _cur_model = current_model _cur_provider = current_provider _cur_base_url = current_base_url _cur_api_key = current_api_key async def _on_model_selected( _chat_id: str, model_id: str, provider_slug: str ) -> str: """Perform the model switch and return confirmation text.""" result = _switch_model( raw_input=model_id, current_provider=_cur_provider, current_model=_cur_model, current_base_url=_cur_base_url, current_api_key=_cur_api_key, is_global=False, explicit_provider=provider_slug, user_providers=user_provs, custom_providers=custom_provs, ) if not result.success: return f"Error: {result.error_message}" # Update cached agent in-place cached_entry = None _cache_lock = getattr(_self, "_agent_cache_lock", None) _cache = getattr(_self, "_agent_cache", None) if _cache_lock and _cache is not None: with _cache_lock: cached_entry = _cache.get(_session_key) if cached_entry and cached_entry[0] is not None: try: cached_entry[0].switch_model( new_model=result.new_model, new_provider=result.target_provider, api_key=result.api_key, base_url=result.base_url, api_mode=result.api_mode, ) except Exception as exc: logger.warning("Picker model switch failed for cached agent: %s", exc) # Store model note + session override if not hasattr(_self, "_pending_model_notes"): _self._pending_model_notes = {} _self._pending_model_notes[_session_key] = ( f"[Note: model was just switched from {_cur_model} to {result.new_model} " f"via {result.provider_label or result.target_provider}. " f"Adjust your self-identification accordingly.]" ) _self._session_model_overrides[_session_key] = { "model": result.new_model, "provider": result.target_provider, "api_key": result.api_key, "base_url": result.base_url, "api_mode": result.api_mode, } # Evict cached agent so the next turn creates a fresh # agent from the override rather than relying on the # stale cache signature to trigger a rebuild. _self._evict_cached_agent(_session_key) # Build confirmation text plabel = result.provider_label or result.target_provider lines = [f"Model switched to `{result.new_model}`"] lines.append(f"Provider: {plabel}") mi = result.model_info from hermes_cli.model_switch import resolve_display_context_length _sw_config_ctx = None try: _sw_cfg = _load_gateway_config() _sw_model_cfg = _sw_cfg.get("model", {}) if isinstance(_sw_model_cfg, dict): _sw_raw = _sw_model_cfg.get("context_length") if _sw_raw is not None: _sw_config_ctx = int(_sw_raw) except Exception: pass ctx = resolve_display_context_length( result.new_model, result.target_provider, base_url=result.base_url or current_base_url or "", api_key=result.api_key or current_api_key or "", model_info=mi, custom_providers=custom_provs, config_context_length=_sw_config_ctx, ) if ctx: lines.append(f"Context: {ctx:,} tokens") if mi: if mi.max_output: lines.append(f"Max output: {mi.max_output:,} tokens") if mi.has_cost_data(): lines.append(f"Cost: {mi.format_cost()}") lines.append(f"Capabilities: {mi.format_capabilities()}") lines.append("_(session only — use `/model --global` to persist)_") return "\n".join(lines) metadata = {"thread_id": source.thread_id} if source.thread_id else None result = await adapter.send_model_picker( chat_id=source.chat_id, providers=providers, current_model=current_model, current_provider=current_provider, session_key=session_key, on_model_selected=_on_model_selected, metadata=metadata, ) if result.success: return None # Picker sent — adapter handles the response # Fallback: text list (for platforms without picker or if picker failed) provider_label = get_label(current_provider) lines = [f"Current: `{current_model or 'unknown'}` on {provider_label}", ""] try: providers = list_authenticated_providers( current_provider=current_provider, current_base_url=current_base_url, current_model=current_model, user_providers=user_provs, custom_providers=custom_provs, max_models=5, ) for p in providers: tag = " (current)" if p["is_current"] else "" lines.append(f"**{p['name']}** `--provider {p['slug']}`{tag}:") if p["models"]: model_strs = ", ".join(f"`{m}`" for m in p["models"]) extra = f" (+{p['total_models'] - len(p['models'])} more)" if p["total_models"] > len(p["models"]) else "" lines.append(f" {model_strs}{extra}") elif p.get("api_url"): lines.append(f" `{p['api_url']}`") lines.append("") except Exception: pass lines.append("`/model ` — switch model") lines.append("`/model --provider ` — switch provider") lines.append("`/model --global` — persist") return "\n".join(lines) # Perform the switch result = _switch_model( raw_input=model_input, current_provider=current_provider, current_model=current_model, current_base_url=current_base_url, current_api_key=current_api_key, is_global=persist_global, explicit_provider=explicit_provider, user_providers=user_provs, custom_providers=custom_provs, ) if not result.success: return f"Error: {result.error_message}" # If there's a cached agent, update it in-place cached_entry = None _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock and _cache is not None: with _cache_lock: cached_entry = _cache.get(session_key) if cached_entry and cached_entry[0] is not None: try: cached_entry[0].switch_model( new_model=result.new_model, new_provider=result.target_provider, api_key=result.api_key, base_url=result.base_url, api_mode=result.api_mode, ) except Exception as exc: logger.warning("In-place model switch failed for cached agent: %s", exc) # Store a note to prepend to the next user message so the model # knows about the switch (avoids system messages mid-history). if not hasattr(self, "_pending_model_notes"): self._pending_model_notes = {} self._pending_model_notes[session_key] = ( f"[Note: model was just switched from {current_model} to {result.new_model} " f"via {result.provider_label or result.target_provider}. " f"Adjust your self-identification accordingly.]" ) # Store session override so next agent creation uses the new model self._session_model_overrides[session_key] = { "model": result.new_model, "provider": result.target_provider, "api_key": result.api_key, "base_url": result.base_url, "api_mode": result.api_mode, } # Evict cached agent so the next turn creates a fresh agent from the # override rather than relying on cache signature mismatch detection. self._evict_cached_agent(session_key) # Persist to config if --global if persist_global: try: if config_path.exists(): with open(config_path, encoding="utf-8") as f: cfg = yaml.safe_load(f) or {} else: cfg = {} model_cfg = cfg.setdefault("model", {}) model_cfg["default"] = result.new_model model_cfg["provider"] = result.target_provider if result.base_url: model_cfg["base_url"] = result.base_url from hermes_cli.config import save_config save_config(cfg) except Exception as e: logger.warning("Failed to persist model switch: %s", e) # Build confirmation message with full metadata provider_label = result.provider_label or result.target_provider lines = [f"Model switched to `{result.new_model}`"] lines.append(f"Provider: {provider_label}") # Context: always resolve via the provider-aware chain so Codex OAuth, # Copilot, and Nous-enforced caps win over the raw models.dev entry. mi = result.model_info from hermes_cli.model_switch import resolve_display_context_length _sw2_config_ctx = None try: _sw2_cfg = _load_gateway_config() _sw2_model_cfg = _sw2_cfg.get("model", {}) if isinstance(_sw2_model_cfg, dict): _sw2_raw = _sw2_model_cfg.get("context_length") if _sw2_raw is not None: _sw2_config_ctx = int(_sw2_raw) except Exception: pass ctx = resolve_display_context_length( result.new_model, result.target_provider, base_url=result.base_url or current_base_url or "", api_key=result.api_key or current_api_key or "", model_info=mi, custom_providers=custom_provs, config_context_length=_sw2_config_ctx, ) if ctx: lines.append(f"Context: {ctx:,} tokens") if mi: if mi.max_output: lines.append(f"Max output: {mi.max_output:,} tokens") if mi.has_cost_data(): lines.append(f"Cost: {mi.format_cost()}") lines.append(f"Capabilities: {mi.format_capabilities()}") # Cache notice cache_enabled = ( (base_url_host_matches(result.base_url or "", "openrouter.ai") and "claude" in result.new_model.lower()) or result.api_mode == "anthropic_messages" ) if cache_enabled: lines.append("Prompt caching: enabled") if result.warning_message: lines.append(f"Warning: {result.warning_message}") if persist_global: lines.append("Saved to config.yaml (`--global`)") else: lines.append("_(session only -- add `--global` to persist)_") return "\n".join(lines) async def _handle_personality_command(self, event: MessageEvent) -> str: """Handle /personality command - list or set a personality.""" from hermes_constants import display_hermes_home args = event.get_command_args().strip().lower() config_path = _hermes_home / 'config.yaml' try: config = _load_gateway_config() personalities = cfg_get(config, "agent", "personalities", default={}) except Exception: config = {} personalities = {} if not personalities: return f"No personalities configured in `{display_hermes_home()}/config.yaml`" if not args: lines = ["🎭 **Available Personalities**\n"] lines.append("• `none` — (no personality overlay)") for name, prompt in personalities.items(): if isinstance(prompt, dict): preview = prompt.get("description") or prompt.get("system_prompt", "")[:50] else: preview = prompt[:50] + "..." if len(prompt) > 50 else prompt lines.append(f"• `{name}` — {preview}") lines.append("\nUsage: `/personality `") return "\n".join(lines) def _resolve_prompt(value): if isinstance(value, dict): parts = [value.get("system_prompt", "")] if value.get("tone"): parts.append(f'Tone: {value["tone"]}') if value.get("style"): parts.append(f'Style: {value["style"]}') return "\n".join(p for p in parts if p) return str(value) if args in ("none", "default", "neutral"): try: if "agent" not in config or not isinstance(config.get("agent"), dict): config["agent"] = {} config["agent"]["system_prompt"] = "" atomic_yaml_write(config_path, config) except Exception as e: return f"⚠️ Failed to save personality change: {e}" self._ephemeral_system_prompt = "" return "🎭 Personality cleared — using base agent behavior.\n_(takes effect on next message)_" elif args in personalities: new_prompt = _resolve_prompt(personalities[args]) # Write to config.yaml, same pattern as CLI save_config_value. try: if "agent" not in config or not isinstance(config.get("agent"), dict): config["agent"] = {} config["agent"]["system_prompt"] = new_prompt atomic_yaml_write(config_path, config) except Exception as e: return f"⚠️ Failed to save personality change: {e}" # Update in-memory so it takes effect on the very next message. self._ephemeral_system_prompt = new_prompt return f"🎭 Personality set to **{args}**\n_(takes effect on next message)_" available = "`none`, " + ", ".join(f"`{n}`" for n in personalities) return f"Unknown personality: `{args}`\n\nAvailable: {available}" async def _handle_retry_command(self, event: MessageEvent) -> str: """Handle /retry command - re-send the last user message.""" source = event.source session_entry = self.session_store.get_or_create_session(source) history = self.session_store.load_transcript(session_entry.session_id) # Find the last user message last_user_msg = None last_user_idx = None for i in range(len(history) - 1, -1, -1): if history[i].get("role") == "user": last_user_msg = history[i].get("content", "") last_user_idx = i break if not last_user_msg: return "No previous message to retry." # Truncate history to before the last user message and persist truncated = history[:last_user_idx] self.session_store.rewrite_transcript(session_entry.session_id, truncated) # Reset stored token count — transcript was truncated session_entry.last_prompt_tokens = 0 # Re-send by creating a fake text event with the old message retry_event = MessageEvent( text=last_user_msg, message_type=MessageType.TEXT, source=source, raw_message=event.raw_message, channel_prompt=event.channel_prompt, ) # Let the normal message handler process it return await self._handle_message(retry_event) # ──────────────────────────────────────────────────────────────── # /goal — persistent cross-turn goals (Ralph-style loop) # ──────────────────────────────────────────────────────────────── def _get_goal_manager_for_event(self, event: "MessageEvent"): """Return a GoalManager bound to the session for this gateway event. Returns ``(manager, session_entry)`` or ``(None, None)`` if the goals module can't be loaded. """ try: from hermes_cli.goals import GoalManager except Exception as exc: logger.debug("goal manager unavailable: %s", exc) return None, None try: session_entry = self.session_store.get_or_create_session(event.source) except Exception as exc: logger.debug("goal manager: session lookup failed: %s", exc) return None, None sid = getattr(session_entry, "session_id", None) or "" if not sid: return None, None try: goals_cfg = ( (self.config or {}).get("goals", {}) if isinstance(self.config, dict) else getattr(self.config, "goals", {}) or {} ) max_turns = int(goals_cfg.get("max_turns", 20) or 20) except Exception: max_turns = 20 return GoalManager(session_id=sid, default_max_turns=max_turns), session_entry async def _handle_goal_command(self, event: "MessageEvent") -> str: """Handle /goal for gateway platforms. Subcommands: ``/goal`` / ``/goal status`` / ``/goal pause`` / ``/goal resume`` / ``/goal clear``. Any other text becomes the new goal. Setting a new goal queues the goal text as the next turn so the agent starts working on it immediately — the post-turn continuation hook then takes over from there. """ args = (event.get_command_args() or "").strip() lower = args.lower() mgr, session_entry = self._get_goal_manager_for_event(event) if mgr is None: return "Goals unavailable on this session." if not args or lower == "status": return mgr.status_line() if lower == "pause": state = mgr.pause(reason="user-paused") if state is None: return "No goal set." return f"⏸ Goal paused: {state.goal}" if lower == "resume": state = mgr.resume() if state is None: return "No goal to resume." return ( f"▶ Goal resumed: {state.goal}\n" "Send any message to continue, or wait — I'll take the next step on the next turn." ) if lower in ("clear", "stop", "done"): had = mgr.has_goal() mgr.clear() return "✓ Goal cleared." if had else "No active goal." # Otherwise — treat the remaining text as the new goal. try: state = mgr.set(args) except ValueError as exc: return f"Invalid goal: {exc}" # Queue the goal text as an immediate first turn so the agent # starts making progress. The post-turn hook takes over after. adapter = self.adapters.get(event.source.platform) if event.source else None _quick_key = self._session_key_for_source(event.source) if event.source else None if adapter and _quick_key: try: kickoff_event = MessageEvent( text=state.goal, message_type=MessageType.TEXT, source=event.source, message_id=event.message_id, channel_prompt=event.channel_prompt, ) self._enqueue_fifo(_quick_key, kickoff_event, adapter) except Exception as exc: logger.debug("goal kickoff enqueue failed: %s", exc) return ( f"⊙ Goal set ({state.max_turns}-turn budget): {state.goal}\n" "I'll keep working until the goal is done, you pause/clear it, or the budget is exhausted.\n" "Controls: /goal status · /goal pause · /goal resume · /goal clear" ) def _post_turn_goal_continuation( self, *, session_entry: Any, source: Any, final_response: str, ) -> None: """Run the goal judge after a gateway turn and, if still active, enqueue a continuation prompt for the same session. Called from ``_handle_message_with_agent`` at turn boundary, AFTER the response has been delivered. Safe when no goal is set. We use the adapter's pending-message / FIFO machinery so any real user message that arrives simultaneously is handled by the same queue and takes priority naturally. """ try: from hermes_cli.goals import GoalManager except Exception as exc: logger.debug("goal continuation: goals module unavailable: %s", exc) return sid = getattr(session_entry, "session_id", None) or "" if not sid: return try: goals_cfg = ( (self.config or {}).get("goals", {}) if isinstance(self.config, dict) else getattr(self.config, "goals", {}) or {} ) max_turns = int(goals_cfg.get("max_turns", 20) or 20) except Exception: max_turns = 20 mgr = GoalManager(session_id=sid, default_max_turns=max_turns) if not mgr.is_active(): return decision = mgr.evaluate_after_turn(final_response or "", user_initiated=True) msg = decision.get("message") or "" # Send the status line back to the user so they see the judge's # verdict. Fire-and-forget via the adapter. if msg and source is not None: try: adapter = self.adapters.get(source.platform) if adapter and hasattr(adapter, "send_message"): import asyncio as _asyncio coro = adapter.send_message(source, msg) if _asyncio.iscoroutine(coro): try: loop = _asyncio.get_event_loop() if loop.is_running(): loop.create_task(coro) else: loop.run_until_complete(coro) except RuntimeError: # No event loop in this thread — schedule on the main one. try: _asyncio.run_coroutine_threadsafe(coro, self._loop) except Exception: pass except Exception as exc: logger.debug("goal continuation: status send failed: %s", exc) if not decision.get("should_continue"): return prompt = decision.get("continuation_prompt") or "" if not prompt or source is None: return # Enqueue via the adapter's FIFO so a user message already in # flight preempts the continuation naturally. try: adapter = self.adapters.get(source.platform) _quick_key = self._session_key_for_source(source) if adapter and _quick_key: cont_event = MessageEvent( text=prompt, message_type=MessageType.TEXT, source=source, message_id=None, channel_prompt=None, ) self._enqueue_fifo(_quick_key, cont_event, adapter) except Exception as exc: logger.debug("goal continuation: enqueue failed: %s", exc) async def _handle_undo_command(self, event: MessageEvent) -> str: """Handle /undo command - remove the last user/assistant exchange.""" source = event.source session_entry = self.session_store.get_or_create_session(source) history = self.session_store.load_transcript(session_entry.session_id) # Find the last user message and remove everything from it onward last_user_idx = None for i in range(len(history) - 1, -1, -1): if history[i].get("role") == "user": last_user_idx = i break if last_user_idx is None: return "Nothing to undo." removed_msg = history[last_user_idx].get("content", "") removed_count = len(history) - last_user_idx self.session_store.rewrite_transcript(session_entry.session_id, history[:last_user_idx]) # Reset stored token count — transcript was truncated session_entry.last_prompt_tokens = 0 preview = removed_msg[:40] + "..." if len(removed_msg) > 40 else removed_msg return f"↩️ Undid {removed_count} message(s).\nRemoved: \"{preview}\"" async def _handle_set_home_command(self, event: MessageEvent) -> str: """Handle /sethome command -- set the current chat as the platform's home channel.""" source = event.source platform_name = source.platform.value if source.platform else "unknown" chat_id = source.chat_id chat_name = source.chat_name or chat_id env_key = _home_target_env_var(platform_name) # Save to .env so it persists across restarts try: from hermes_cli.config import save_env_value save_env_value(env_key, str(chat_id)) except Exception as e: return f"Failed to save home channel: {e}" return ( f"✅ Home channel set to **{chat_name}** (ID: {chat_id}).\n" f"Cron jobs and cross-platform messages will be delivered here." ) @staticmethod def _get_guild_id(event: MessageEvent) -> Optional[int]: """Extract Discord guild_id from the raw message object.""" raw = getattr(event, "raw_message", None) if raw is None: return None # Slash command interaction if hasattr(raw, "guild_id") and raw.guild_id: return int(raw.guild_id) # Regular message if hasattr(raw, "guild") and raw.guild: return raw.guild.id return None async def _handle_voice_command(self, event: MessageEvent) -> str: """Handle /voice [on|off|tts|channel|leave|status] command.""" args = event.get_command_args().strip().lower() chat_id = event.source.chat_id platform = event.source.platform voice_key = self._voice_key(platform, chat_id) adapter = self.adapters.get(platform) if args in ("on", "enable"): self._voice_mode[voice_key] = "voice_only" self._save_voice_modes() if adapter: self._set_adapter_auto_tts_enabled(adapter, chat_id, enabled=True) return ( "Voice mode enabled.\n" "I'll reply with voice when you send voice messages.\n" "Use /voice tts to get voice replies for all messages." ) elif args in ("off", "disable"): self._voice_mode[voice_key] = "off" self._save_voice_modes() if adapter: self._set_adapter_auto_tts_disabled(adapter, chat_id, disabled=True) return "Voice mode disabled. Text-only replies." elif args == "tts": self._voice_mode[voice_key] = "all" self._save_voice_modes() if adapter: self._set_adapter_auto_tts_enabled(adapter, chat_id, enabled=True) return ( "Auto-TTS enabled.\n" "All replies will include a voice message." ) elif args in ("channel", "join"): return await self._handle_voice_channel_join(event) elif args == "leave": return await self._handle_voice_channel_leave(event) elif args == "status": mode = self._voice_mode.get(voice_key, "off") labels = { "off": "Off (text only)", "voice_only": "On (voice reply to voice messages)", "all": "TTS (voice reply to all messages)", } # Append voice channel info if connected adapter = self.adapters.get(event.source.platform) guild_id = self._get_guild_id(event) if guild_id and hasattr(adapter, "get_voice_channel_info"): info = adapter.get_voice_channel_info(guild_id) if info: lines = [ f"Voice mode: {labels.get(mode, mode)}", f"Voice channel: #{info['channel_name']}", f"Participants: {info['member_count']}", ] for m in info["members"]: status = " (speaking)" if m.get("is_speaking") else "" lines.append(f" - {m['display_name']}{status}") return "\n".join(lines) return f"Voice mode: {labels.get(mode, mode)}" else: # Toggle: off → on, on/all → off current = self._voice_mode.get(voice_key, "off") if current == "off": self._voice_mode[voice_key] = "voice_only" self._save_voice_modes() if adapter: self._set_adapter_auto_tts_enabled(adapter, chat_id, enabled=True) return "Voice mode enabled." else: self._voice_mode[voice_key] = "off" self._save_voice_modes() if adapter: self._set_adapter_auto_tts_disabled(adapter, chat_id, disabled=True) return "Voice mode disabled." async def _handle_voice_channel_join(self, event: MessageEvent) -> str: """Join the user's current Discord voice channel.""" adapter = self.adapters.get(event.source.platform) if not hasattr(adapter, "join_voice_channel"): return "Voice channels are not supported on this platform." guild_id = self._get_guild_id(event) if not guild_id: return "This command only works in a Discord server." voice_channel = await adapter.get_user_voice_channel( guild_id, event.source.user_id ) if not voice_channel: return "You need to be in a voice channel first." # Wire callbacks BEFORE join so voice input arriving immediately # after connection is not lost. if hasattr(adapter, "_voice_input_callback"): adapter._voice_input_callback = self._handle_voice_channel_input if hasattr(adapter, "_on_voice_disconnect"): adapter._on_voice_disconnect = self._handle_voice_timeout_cleanup try: success = await adapter.join_voice_channel(voice_channel) except Exception as e: logger.warning("Failed to join voice channel: %s", e) adapter._voice_input_callback = None err_lower = str(e).lower() if "pynacl" in err_lower or "nacl" in err_lower or "davey" in err_lower: return ( "Voice dependencies are missing (PyNaCl / davey). " f"Install with: `{sys.executable} -m pip install PyNaCl`" ) return f"Failed to join voice channel: {e}" if success: adapter._voice_text_channels[guild_id] = int(event.source.chat_id) if hasattr(adapter, "_voice_sources"): adapter._voice_sources[guild_id] = event.source.to_dict() self._voice_mode[self._voice_key(event.source.platform, event.source.chat_id)] = "all" self._save_voice_modes() self._set_adapter_auto_tts_enabled(adapter, event.source.chat_id, enabled=True) return ( f"Joined voice channel **{voice_channel.name}**.\n" f"I'll speak my replies and listen to you. Use /voice leave to disconnect." ) # Join failed — clear callback adapter._voice_input_callback = None return "Failed to join voice channel. Check bot permissions (Connect + Speak)." async def _handle_voice_channel_leave(self, event: MessageEvent) -> str: """Leave the Discord voice channel.""" adapter = self.adapters.get(event.source.platform) guild_id = self._get_guild_id(event) if not guild_id or not hasattr(adapter, "leave_voice_channel"): return "Not in a voice channel." if not hasattr(adapter, "is_in_voice_channel") or not adapter.is_in_voice_channel(guild_id): return "Not in a voice channel." try: await adapter.leave_voice_channel(guild_id) except Exception as e: logger.warning("Error leaving voice channel: %s", e) # Always clean up state even if leave raised an exception self._voice_mode[self._voice_key(event.source.platform, event.source.chat_id)] = "off" self._save_voice_modes() self._set_adapter_auto_tts_disabled(adapter, event.source.chat_id, disabled=True) if hasattr(adapter, "_voice_input_callback"): adapter._voice_input_callback = None return "Left voice channel." def _handle_voice_timeout_cleanup(self, chat_id: str) -> None: """Called by the adapter when a voice channel times out. Cleans up runner-side voice_mode state that the adapter cannot reach. """ self._voice_mode[self._voice_key(Platform.DISCORD, chat_id)] = "off" self._save_voice_modes() adapter = self.adapters.get(Platform.DISCORD) self._set_adapter_auto_tts_disabled(adapter, chat_id, disabled=True) async def _handle_voice_channel_input( self, guild_id: int, user_id: int, transcript: str ): """Handle transcribed voice from a user in a voice channel. Creates a synthetic MessageEvent and processes it through the adapter's full message pipeline (session, typing, agent, TTS reply). """ adapter = self.adapters.get(Platform.DISCORD) if not adapter: return text_ch_id = adapter._voice_text_channels.get(guild_id) if not text_ch_id: return # Build source — reuse the linked text channel's metadata when available # so voice input shares the same session as the bound text conversation. source_data = getattr(adapter, "_voice_sources", {}).get(guild_id) if source_data: source = SessionSource.from_dict(source_data) source.user_id = str(user_id) source.user_name = str(user_id) else: source = SessionSource( platform=Platform.DISCORD, chat_id=str(text_ch_id), user_id=str(user_id), user_name=str(user_id), chat_type="channel", ) # Check authorization before processing voice input if not self._is_user_authorized(source): logger.debug("Unauthorized voice input from user %d, ignoring", user_id) return # Show transcript in text channel (after auth, with mention sanitization) try: channel = adapter._client.get_channel(text_ch_id) if channel: safe_text = transcript[:2000].replace("@everyone", "@\u200beveryone").replace("@here", "@\u200bhere") await channel.send(f"**[Voice]** <@{user_id}>: {safe_text}") except Exception: pass # Build a synthetic MessageEvent and feed through the normal pipeline # Use SimpleNamespace as raw_message so _get_guild_id() can extract # guild_id and _send_voice_reply() plays audio in the voice channel. from types import SimpleNamespace event = MessageEvent( source=source, text=transcript, message_type=MessageType.VOICE, raw_message=SimpleNamespace(guild_id=guild_id, guild=None), ) await adapter.handle_message(event) def _should_send_voice_reply( self, event: MessageEvent, response: str, agent_messages: list, already_sent: bool = False, ) -> bool: """Decide whether the runner should send a TTS voice reply. Returns False when: - voice_mode is off for this chat - response is empty or an error - agent already called text_to_speech tool (dedup) - voice input and base adapter auto-TTS already handled it (skip_double) UNLESS streaming already consumed the response (already_sent=True), in which case the base adapter won't have text for auto-TTS so the runner must handle it. """ if not response or response.startswith("Error:"): return False chat_id = event.source.chat_id voice_mode = self._voice_mode.get(self._voice_key(event.source.platform, chat_id), "off") is_voice_input = (event.message_type == MessageType.VOICE) should = ( (voice_mode == "all") or (voice_mode == "voice_only" and is_voice_input) ) if not should: return False # Dedup: agent already called TTS tool has_agent_tts = any( msg.get("role") == "assistant" and any( tc.get("function", {}).get("name") == "text_to_speech" for tc in (msg.get("tool_calls") or []) ) for msg in agent_messages ) if has_agent_tts: return False # Dedup: base adapter auto-TTS already handles voice input # (play_tts plays in VC when connected, so runner can skip). # When streaming already delivered the text (already_sent=True), # the base adapter will receive None and can't run auto-TTS, # so the runner must take over. if is_voice_input and not already_sent: return False return True async def _send_voice_reply(self, event: MessageEvent, text: str) -> None: """Generate TTS audio and send as a voice message before the text reply.""" import uuid as _uuid audio_path = None actual_path = None try: from tools.tts_tool import text_to_speech_tool, _strip_markdown_for_tts tts_text = _strip_markdown_for_tts(text[:4000]) if not tts_text: return # Use .mp3 extension so edge-tts conversion to opus works correctly. # The TTS tool may convert to .ogg — use file_path from result. audio_path = os.path.join( tempfile.gettempdir(), "hermes_voice", f"tts_reply_{_uuid.uuid4().hex[:12]}.mp3", ) os.makedirs(os.path.dirname(audio_path), exist_ok=True) result_json = await asyncio.to_thread( text_to_speech_tool, text=tts_text, output_path=audio_path ) result = json.loads(result_json) # Use the actual file path from result (may differ after opus conversion) actual_path = result.get("file_path", audio_path) if not result.get("success") or not os.path.isfile(actual_path): logger.warning("Auto voice reply TTS failed: %s", result.get("error")) return adapter = self.adapters.get(event.source.platform) # If connected to a voice channel, play there instead of sending a file guild_id = self._get_guild_id(event) if (guild_id and hasattr(adapter, "play_in_voice_channel") and hasattr(adapter, "is_in_voice_channel") and adapter.is_in_voice_channel(guild_id)): await adapter.play_in_voice_channel(guild_id, actual_path) elif adapter and hasattr(adapter, "send_voice"): send_kwargs: Dict[str, Any] = { "chat_id": event.source.chat_id, "audio_path": actual_path, "reply_to": event.message_id, } if event.source.thread_id: send_kwargs["metadata"] = {"thread_id": event.source.thread_id} await adapter.send_voice(**send_kwargs) except Exception as e: logger.warning("Auto voice reply failed: %s", e, exc_info=True) finally: for p in {audio_path, actual_path} - {None}: try: os.unlink(p) except OSError: pass async def _deliver_media_from_response( self, response: str, event: MessageEvent, adapter, ) -> None: """Extract MEDIA: tags and local file paths from a response and deliver them. Called after streaming has already sent the text to the user, so the text itself is already delivered — this only handles file attachments that the normal _process_message_background path would have caught. """ from pathlib import Path from urllib.parse import quote as _quote try: media_files, _ = adapter.extract_media(response) _, cleaned = adapter.extract_images(response) local_files, _ = adapter.extract_local_files(cleaned) _thread_meta = {"thread_id": event.source.thread_id} if event.source.thread_id else None from gateway.platforms.base import should_send_media_as_audio _VIDEO_EXTS = {'.mp4', '.mov', '.avi', '.mkv', '.webm', '.3gp'} _IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.webp', '.gif'} # Partition out images so they can be sent as a single batch # (e.g. Signal's multi-attachment RPC) image_paths: list = [] non_image_media: list = [] for media_path, is_voice in media_files: ext = Path(media_path).suffix.lower() if ext in _IMAGE_EXTS and not is_voice: image_paths.append(media_path) else: non_image_media.append((media_path, is_voice)) non_image_local: list = [] for file_path in local_files: if Path(file_path).suffix.lower() in _IMAGE_EXTS: image_paths.append(file_path) else: non_image_local.append(file_path) if image_paths: try: images = [(f"file://{_quote(p)}", "") for p in image_paths] await adapter.send_multiple_images( chat_id=event.source.chat_id, images=images, metadata=_thread_meta, ) except Exception as e: logger.warning("[%s] Post-stream image batch delivery failed: %s", adapter.name, e) for media_path, is_voice in non_image_media: try: ext = Path(media_path).suffix.lower() if should_send_media_as_audio(event.source.platform, ext, is_voice=is_voice): await adapter.send_voice( chat_id=event.source.chat_id, audio_path=media_path, metadata=_thread_meta, ) elif ext in _VIDEO_EXTS: await adapter.send_video( chat_id=event.source.chat_id, video_path=media_path, metadata=_thread_meta, ) else: await adapter.send_document( chat_id=event.source.chat_id, file_path=media_path, metadata=_thread_meta, ) except Exception as e: logger.warning("[%s] Post-stream media delivery failed: %s", adapter.name, e) for file_path in non_image_local: try: ext = Path(file_path).suffix.lower() if ext in _VIDEO_EXTS: await adapter.send_video( chat_id=event.source.chat_id, video_path=file_path, metadata=_thread_meta, ) else: await adapter.send_document( chat_id=event.source.chat_id, file_path=file_path, metadata=_thread_meta, ) except Exception as e: logger.warning("[%s] Post-stream file delivery failed: %s", adapter.name, e) except Exception as e: logger.warning("Post-stream media extraction failed: %s", e) async def _handle_rollback_command(self, event: MessageEvent) -> str: """Handle /rollback command — list or restore filesystem checkpoints.""" from tools.checkpoint_manager import CheckpointManager, format_checkpoint_list # Read checkpoint config from config.yaml cp_cfg = {} try: import yaml as _y _cfg_path = _hermes_home / "config.yaml" if _cfg_path.exists(): with open(_cfg_path, encoding="utf-8") as _f: _data = _y.safe_load(_f) or {} cp_cfg = _data.get("checkpoints", {}) if isinstance(cp_cfg, bool): cp_cfg = {"enabled": cp_cfg} except Exception: pass if not cp_cfg.get("enabled", False): return ( "Checkpoints are not enabled.\n" "Enable in config.yaml:\n```\ncheckpoints:\n enabled: true\n```" ) mgr = CheckpointManager( enabled=True, max_snapshots=cp_cfg.get("max_snapshots", 50), ) cwd = os.getenv("TERMINAL_CWD", str(Path.home())) arg = event.get_command_args().strip() if not arg: checkpoints = mgr.list_checkpoints(cwd) return format_checkpoint_list(checkpoints, cwd) # Restore by number or hash checkpoints = mgr.list_checkpoints(cwd) if not checkpoints: return f"No checkpoints found for {cwd}" target_hash = None try: idx = int(arg) - 1 if 0 <= idx < len(checkpoints): target_hash = checkpoints[idx]["hash"] else: return f"Invalid checkpoint number. Use 1-{len(checkpoints)}." except ValueError: target_hash = arg result = mgr.restore(cwd, target_hash) if result["success"]: return ( f"✅ Restored to checkpoint {result['restored_to']}: {result['reason']}\n" f"A pre-rollback snapshot was saved automatically." ) return f"❌ {result['error']}" async def _handle_background_command(self, event: MessageEvent) -> str: """Handle /background — run a prompt in a separate background session. Spawns a new AIAgent in a background thread with its own session. When it completes, sends the result back to the same chat without modifying the active session's conversation history. """ prompt = event.get_command_args().strip() if not prompt: return ( "Usage: /background \n" "Example: /background Summarize the top HN stories today\n\n" "Runs the prompt in a separate session. " "You can keep chatting — the result will appear here when done." ) source = event.source task_id = f"bg_{datetime.now().strftime('%H%M%S')}_{os.urandom(3).hex()}" # Fire-and-forget the background task _task = asyncio.create_task( self._run_background_task(prompt, source, task_id) ) self._background_tasks.add(_task) _task.add_done_callback(self._background_tasks.discard) preview = prompt[:60] + ("..." if len(prompt) > 60 else "") return f'🔄 Background task started: "{preview}"\nTask ID: {task_id}\nYou can keep chatting — results will appear when done.' async def _run_background_task( self, prompt: str, source: "SessionSource", task_id: str ) -> None: """Execute a background agent task and deliver the result to the chat.""" from run_agent import AIAgent adapter = self.adapters.get(source.platform) if not adapter: logger.warning("No adapter for platform %s in background task %s", source.platform, task_id) return _thread_metadata = {"thread_id": source.thread_id} if source.thread_id else None try: user_config = _load_gateway_config() model, runtime_kwargs = self._resolve_session_agent_runtime( source=source, user_config=user_config, ) if not runtime_kwargs.get("api_key"): await adapter.send( source.chat_id, f"❌ Background task {task_id} failed: no provider credentials configured.", metadata=_thread_metadata, ) return platform_key = _platform_config_key(source.platform) from hermes_cli.tools_config import _get_platform_tools enabled_toolsets = sorted(_get_platform_tools(user_config, platform_key)) agent_cfg = user_config.get("agent") or {} disabled_toolsets = agent_cfg.get("disabled_toolsets") or None pr = self._provider_routing max_iterations = int(os.getenv("HERMES_MAX_ITERATIONS", "90")) reasoning_config = self._resolve_session_reasoning_config(source=source) self._reasoning_config = reasoning_config self._service_tier = self._load_service_tier() turn_route = self._resolve_turn_agent_config(prompt, model, runtime_kwargs) def run_sync(): agent = AIAgent( model=turn_route["model"], **turn_route["runtime"], max_iterations=max_iterations, quiet_mode=True, verbose_logging=False, enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, reasoning_config=reasoning_config, service_tier=self._service_tier, request_overrides=turn_route.get("request_overrides"), providers_allowed=pr.get("only"), providers_ignored=pr.get("ignore"), providers_order=pr.get("order"), provider_sort=pr.get("sort"), provider_require_parameters=pr.get("require_parameters", False), provider_data_collection=pr.get("data_collection"), session_id=task_id, platform=platform_key, user_id=source.user_id, user_name=source.user_name, chat_id=source.chat_id, chat_name=source.chat_name, chat_type=source.chat_type, thread_id=source.thread_id, session_db=self._session_db, fallback_model=self._fallback_model, ) try: return agent.run_conversation( user_message=prompt, task_id=task_id, ) finally: self._cleanup_agent_resources(agent) result = await self._run_in_executor_with_context(run_sync) response = result.get("final_response", "") if result else "" if not response and result and result.get("error"): response = f"Error: {result['error']}" # Extract media files from the response if response: media_files, response = adapter.extract_media(response) images, text_content = adapter.extract_images(response) preview = prompt[:60] + ("..." if len(prompt) > 60 else "") header = f'✅ Background task complete\nPrompt: "{preview}"\n\n' if text_content: await adapter.send( chat_id=source.chat_id, content=header + text_content, metadata=_thread_metadata, ) elif not images and not media_files: await adapter.send( chat_id=source.chat_id, content=header + "(No response generated)", metadata=_thread_metadata, ) # Send extracted images for image_url, alt_text in (images or []): try: await adapter.send_image( chat_id=source.chat_id, image_url=image_url, caption=alt_text, metadata=_thread_metadata, ) except Exception: pass # Send media files for media_path, _is_voice in (media_files or []): try: await adapter.send_document( chat_id=source.chat_id, file_path=media_path, metadata=_thread_metadata, ) except Exception: pass else: preview = prompt[:60] + ("..." if len(prompt) > 60 else "") await adapter.send( chat_id=source.chat_id, content=f'✅ Background task complete\nPrompt: "{preview}"\n\n(No response generated)', metadata=_thread_metadata, ) except Exception as e: logger.exception("Background task %s failed", task_id) try: await adapter.send( chat_id=source.chat_id, content=f"❌ Background task {task_id} failed: {e}", metadata=_thread_metadata, ) except Exception: pass async def _handle_reasoning_command(self, event: MessageEvent) -> str: """Handle /reasoning command — manage reasoning effort and display toggle. Usage: /reasoning Show current effort level and display state /reasoning Set reasoning effort for this session only /reasoning --global Persist reasoning effort to config.yaml /reasoning reset Clear this session's reasoning override /reasoning show|on Show model reasoning in responses /reasoning hide|off Hide model reasoning from responses """ import yaml raw_args = event.get_command_args().strip() args, persist_global = self._parse_reasoning_command_args(raw_args) config_path = _hermes_home / "config.yaml" session_key = self._session_key_for_source(event.source) self._show_reasoning = self._load_show_reasoning() self._reasoning_config = self._resolve_session_reasoning_config( source=event.source, session_key=session_key, ) def _save_config_key(key_path: str, value): """Save a dot-separated key to config.yaml.""" try: user_config = {} if config_path.exists(): with open(config_path, encoding="utf-8") as f: user_config = yaml.safe_load(f) or {} keys = key_path.split(".") current = user_config for k in keys[:-1]: if k not in current or not isinstance(current[k], dict): current[k] = {} current = current[k] current[keys[-1]] = value atomic_yaml_write(config_path, user_config) return True except Exception as e: logger.error("Failed to save config key %s: %s", key_path, e) return False if not raw_args: # Show current state rc = self._reasoning_config if rc is None: level = "medium (default)" elif rc.get("enabled") is False: level = "none (disabled)" else: level = rc.get("effort", "medium") display_state = "on ✓" if self._show_reasoning else "off" has_session_override = session_key in (getattr(self, "_session_reasoning_overrides", {}) or {}) scope = "session override" if has_session_override else "global config" return ( "🧠 **Reasoning Settings**\n\n" f"**Effort:** `{level}`\n" f"**Scope:** {scope}\n" f"**Display:** {display_state}\n\n" "_Usage:_ `/reasoning [--global]`" ) # Display toggle (per-platform) platform_key = _platform_config_key(event.source.platform) if args in ("show", "on"): self._show_reasoning = True _save_config_key(f"display.platforms.{platform_key}.show_reasoning", True) return ( "🧠 ✓ Reasoning display: **ON**\n" f"Model thinking will be shown before each response on **{platform_key}**." ) if args in ("hide", "off"): self._show_reasoning = False _save_config_key(f"display.platforms.{platform_key}.show_reasoning", False) return f"🧠 ✓ Reasoning display: **OFF** for **{platform_key}**" # Effort level change effort = args.strip() if effort == "reset": if persist_global: return "⚠️ `/reasoning reset --global` is not supported. Use `/reasoning --global` to change the global default." self._set_session_reasoning_override(session_key, None) self._reasoning_config = self._load_reasoning_config() self._evict_cached_agent(session_key) return "🧠 ✓ Session reasoning override cleared; falling back to global config." if effort == "none": parsed = {"enabled": False} elif effort in ("minimal", "low", "medium", "high", "xhigh"): parsed = {"enabled": True, "effort": effort} else: return ( f"⚠️ Unknown argument: `{effort or raw_args.lower()}`\n\n" "**Valid levels:** none, minimal, low, medium, high, xhigh\n" "**Display:** show, hide\n" "**Persist:** add `--global` to save beyond this session" ) self._reasoning_config = parsed if persist_global: if _save_config_key("agent.reasoning_effort", effort): self._set_session_reasoning_override(session_key, None) self._evict_cached_agent(session_key) return f"🧠 ✓ Reasoning effort set to `{effort}` (saved to config)\n_(takes effect on next message)_" self._set_session_reasoning_override(session_key, parsed) self._evict_cached_agent(session_key) return f"🧠 ✓ Reasoning effort set to `{effort}` (session only — config save failed)\n_(takes effect on next message)_" self._set_session_reasoning_override(session_key, parsed) self._evict_cached_agent(session_key) return f"🧠 ✓ Reasoning effort set to `{effort}` (session only — add `--global` to persist)\n_(takes effect on next message)_" async def _handle_fast_command(self, event: MessageEvent) -> str: """Handle /fast — mirror the CLI Priority Processing toggle in gateway chats.""" import yaml from hermes_cli.models import model_supports_fast_mode args = event.get_command_args().strip().lower() config_path = _hermes_home / "config.yaml" self._service_tier = self._load_service_tier() user_config = _load_gateway_config() model = _resolve_gateway_model(user_config) if not model_supports_fast_mode(model): return "⚡ /fast is only available for OpenAI models that support Priority Processing." def _save_config_key(key_path: str, value): """Save a dot-separated key to config.yaml.""" try: user_config = {} if config_path.exists(): with open(config_path, encoding="utf-8") as f: user_config = yaml.safe_load(f) or {} keys = key_path.split(".") current = user_config for k in keys[:-1]: if k not in current or not isinstance(current[k], dict): current[k] = {} current = current[k] current[keys[-1]] = value atomic_yaml_write(config_path, user_config) return True except Exception as e: logger.error("Failed to save config key %s: %s", key_path, e) return False if not args or args == "status": status = "fast" if self._service_tier == "priority" else "normal" return ( "⚡ Priority Processing\n\n" f"Current mode: `{status}`\n\n" "_Usage:_ `/fast `" ) if args in {"fast", "on"}: self._service_tier = "priority" saved_value = "fast" label = "FAST" elif args in {"normal", "off"}: self._service_tier = None saved_value = "normal" label = "NORMAL" else: return ( f"⚠️ Unknown argument: `{args}`\n\n" "**Valid options:** normal, fast, status" ) if _save_config_key("agent.service_tier", saved_value): return f"⚡ ✓ Priority Processing: **{label}** (saved to config)\n_(takes effect on next message)_" return f"⚡ ✓ Priority Processing: **{label}** (this session only)" async def _handle_yolo_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: """Handle /yolo — toggle dangerous command approval bypass for this session only.""" from tools.approval import ( disable_session_yolo, enable_session_yolo, is_session_yolo_enabled, ) session_key = self._session_key_for_source(event.source) current = is_session_yolo_enabled(session_key) if current: disable_session_yolo(session_key) return EphemeralReply("⚠️ YOLO mode **OFF** for this session — dangerous commands will require approval.") else: enable_session_yolo(session_key) return EphemeralReply("⚡ YOLO mode **ON** for this session — all commands auto-approved. Use with caution.") async def _handle_verbose_command(self, event: MessageEvent) -> str: """Handle /verbose command — cycle tool progress display mode. Gated by ``display.tool_progress_command`` in config.yaml (default off). When enabled, cycles the tool progress mode through off → new → all → verbose → off for the *current platform*. The setting is saved to ``display.platforms..tool_progress`` so each channel can have its own verbosity level independently. """ config_path = _hermes_home / "config.yaml" platform_key = _platform_config_key(event.source.platform) # --- check config gate ------------------------------------------------ try: user_config = _load_gateway_config() gate_enabled = is_truthy_value( cfg_get(user_config, "display", "tool_progress_command"), default=False, ) except Exception: gate_enabled = False if not gate_enabled: return ( "The `/verbose` command is not enabled for messaging platforms.\n\n" "Enable it in `config.yaml`:\n```yaml\n" "display:\n tool_progress_command: true\n```" ) # --- cycle mode (per-platform) ---------------------------------------- cycle = ["off", "new", "all", "verbose"] descriptions = { "off": "⚙️ Tool progress: **OFF** — no tool activity shown.", "new": "⚙️ Tool progress: **NEW** — shown when tool changes (preview length: `display.tool_preview_length`, default 40).", "all": "⚙️ Tool progress: **ALL** — every tool call shown (preview length: `display.tool_preview_length`, default 40).", "verbose": "⚙️ Tool progress: **VERBOSE** — every tool call with full arguments.", } # Read current effective mode for this platform via the resolver from gateway.display_config import resolve_display_setting current = resolve_display_setting(user_config, platform_key, "tool_progress", "all") if current not in cycle: current = "all" idx = (cycle.index(current) + 1) % len(cycle) new_mode = cycle[idx] # Save to display.platforms..tool_progress try: if "display" not in user_config or not isinstance(user_config.get("display"), dict): user_config["display"] = {} display = user_config["display"] if "platforms" not in display or not isinstance(display.get("platforms"), dict): display["platforms"] = {} if platform_key not in display["platforms"] or not isinstance(display["platforms"].get(platform_key), dict): display["platforms"][platform_key] = {} display["platforms"][platform_key]["tool_progress"] = new_mode atomic_yaml_write(config_path, user_config) return ( f"{descriptions[new_mode]}\n" f"_(saved for **{platform_key}** — takes effect on next message)_" ) except Exception as e: logger.warning("Failed to save tool_progress mode: %s", e) return f"{descriptions[new_mode]}\n_(could not save to config: {e})_" async def _handle_footer_command(self, event: MessageEvent) -> str: """Handle /footer command — toggle the runtime-metadata footer. Usage: /footer → toggle on/off /footer on → enable globally /footer off → disable globally /footer status → show current state + fields The footer is saved to ``display.runtime_footer.enabled`` (global). Per-platform overrides under ``display.platforms..runtime_footer`` are respected but not modified here — edit config.yaml directly for per-platform control. """ from gateway.runtime_footer import resolve_footer_config config_path = _hermes_home / "config.yaml" platform_key = _platform_config_key(event.source.platform) # --- parse argument ------------------------------------------------- arg = "" try: text = (getattr(event, "message", None) or "").strip() if text.startswith("/"): parts = text.split(None, 1) if len(parts) > 1: arg = parts[1].strip().lower() except Exception: arg = "" # --- load config ---------------------------------------------------- try: user_config: dict = _load_gateway_config() except Exception as e: return f"⚠️ Could not read config.yaml: {e}" effective = resolve_footer_config(user_config, platform_key) if arg in ("status", "?"): state = "ON" if effective["enabled"] else "OFF" fields = ", ".join(effective.get("fields") or []) return ( f"📎 Runtime footer: **{state}**\n" f"Fields: `{fields}`\n" f"Platform: `{platform_key}`" ) if arg in ("on", "enable", "true", "1"): new_state = True elif arg in ("off", "disable", "false", "0"): new_state = False elif arg == "": new_state = not effective["enabled"] else: return "Usage: `/footer [on|off|status]`" # --- write global flag --------------------------------------------- try: if not isinstance(user_config.get("display"), dict): user_config["display"] = {} display = user_config["display"] if not isinstance(display.get("runtime_footer"), dict): display["runtime_footer"] = {} display["runtime_footer"]["enabled"] = new_state atomic_yaml_write(config_path, user_config) except Exception as e: logger.warning("Failed to save runtime_footer.enabled: %s", e) return f"⚠️ Could not save config: {e}" state = "ON" if new_state else "OFF" example = "" if new_state: # Show a preview using current agent state if available. from gateway.runtime_footer import format_runtime_footer preview = format_runtime_footer( model=_resolve_gateway_model(user_config) or None, context_tokens=0, context_length=None, fields=effective.get("fields") or ["model", "context_pct", "cwd"], ) if preview: example = f"\nExample: `{preview}`" return ( f"📎 Runtime footer: **{state}**" f"{example}\n" f"_(saved globally — takes effect on next message)_" ) async def _handle_compress_command(self, event: MessageEvent) -> str: """Handle /compress command -- manually compress conversation context. Accepts an optional focus topic: ``/compress `` guides the summariser to preserve information related to *focus* while being more aggressive about discarding everything else. """ source = event.source session_entry = self.session_store.get_or_create_session(source) history = self.session_store.load_transcript(session_entry.session_id) if not history or len(history) < 4: return "Not enough conversation to compress (need at least 4 messages)." # Extract optional focus topic from command args focus_topic = (event.get_command_args() or "").strip() or None try: from run_agent import AIAgent from agent.manual_compression_feedback import summarize_manual_compression from agent.model_metadata import estimate_request_tokens_rough session_key = self._session_key_for_source(source) model, runtime_kwargs = self._resolve_session_agent_runtime( source=source, session_key=session_key, ) if not runtime_kwargs.get("api_key"): return "No provider configured -- cannot compress." msgs = [ {"role": m.get("role"), "content": m.get("content")} for m in history if m.get("role") in ("user", "assistant") and m.get("content") ] tmp_agent = AIAgent( **runtime_kwargs, model=model, max_iterations=4, quiet_mode=True, skip_memory=True, enabled_toolsets=["memory"], session_id=session_entry.session_id, ) try: tmp_agent._print_fn = lambda *a, **kw: None # Estimate with system prompt + tool schemas included so the # figure reflects real request pressure, not a transcript-only # underestimate (#6217). Must be computed after tmp_agent is # built so _cached_system_prompt/tools are populated. _sys_prompt = getattr(tmp_agent, "_cached_system_prompt", "") or "" _tools = getattr(tmp_agent, "tools", None) or None approx_tokens = estimate_request_tokens_rough( msgs, system_prompt=_sys_prompt, tools=_tools ) compressor = tmp_agent.context_compressor if not compressor.has_content_to_compress(msgs): return "Nothing to compress yet (the transcript is still all protected context)." loop = asyncio.get_running_loop() compressed, _ = await loop.run_in_executor( None, lambda: tmp_agent._compress_context(msgs, "", approx_tokens=approx_tokens, focus_topic=focus_topic) ) # _compress_context already calls end_session() on the old session # (preserving its full transcript in SQLite) and creates a new # session_id for the continuation. Write the compressed messages # into the NEW session so the original history stays searchable. new_session_id = tmp_agent.session_id if new_session_id != session_entry.session_id: session_entry.session_id = new_session_id self.session_store._save() self.session_store.rewrite_transcript(new_session_id, compressed) # Reset stored token count — transcript changed, old value is stale self.session_store.update_session( session_entry.session_key, last_prompt_tokens=0 ) new_tokens = estimate_request_tokens_rough( compressed, system_prompt=_sys_prompt, tools=_tools ) summary = summarize_manual_compression( msgs, compressed, approx_tokens, new_tokens, ) # Detect summary-generation failure so we can surface a # visible warning to the user even on the manual /compress # path (otherwise the failure is silently logged). _summary_failed = bool(getattr(compressor, "_last_summary_fallback_used", False)) _dropped_count = int(getattr(compressor, "_last_summary_dropped_count", 0) or 0) _summary_err = getattr(compressor, "_last_summary_error", None) # Separately: did the user's CONFIGURED aux model fail # and we recovered via main? Surface that as an info # note so they can fix their config. _aux_fail_model = getattr(compressor, "_last_aux_model_failure_model", None) _aux_fail_err = getattr(compressor, "_last_aux_model_failure_error", None) finally: self._cleanup_agent_resources(tmp_agent) lines = [f"🗜️ {summary['headline']}"] if focus_topic: lines.append(f"Focus: \"{focus_topic}\"") lines.append(summary["token_line"]) if summary["note"]: lines.append(summary["note"]) if _summary_failed: lines.append( f"⚠️ Summary generation failed ({_summary_err or 'unknown error'}). " f"{_dropped_count} historical message(s) were removed and replaced " "with a placeholder; earlier context is no longer recoverable. " "Consider checking your auxiliary.compression model configuration." ) elif _aux_fail_model: lines.append( f"ℹ️ Configured compression model `{_aux_fail_model}` failed " f"({_aux_fail_err or 'unknown error'}). Recovered using your main " "model — context is intact — but you may want to check " "`auxiliary.compression.model` in config.yaml." ) return "\n".join(lines) except Exception as e: logger.warning("Manual compress failed: %s", e) return f"Compression failed: {e}" async def _handle_title_command(self, event: MessageEvent) -> str: """Handle /title command — set or show the current session's title.""" source = event.source session_entry = self.session_store.get_or_create_session(source) session_id = session_entry.session_id if not self._session_db: return "Session database not available." # Ensure session exists in SQLite DB (it may only exist in session_store # if this is the first command in a new session) existing_title = self._session_db.get_session_title(session_id) if existing_title is None: # Session doesn't exist in DB yet — create it try: self._session_db.create_session( session_id=session_id, source=source.platform.value if source.platform else "unknown", user_id=source.user_id, ) except Exception: pass # Session might already exist, ignore errors title_arg = event.get_command_args().strip() if title_arg: # Sanitize the title before setting try: sanitized = self._session_db.sanitize_title(title_arg) except ValueError as e: return f"⚠️ {e}" if not sanitized: return "⚠️ Title is empty after cleanup. Please use printable characters." # Set the title try: if self._session_db.set_session_title(session_id, sanitized): return f"✏️ Session title set: **{sanitized}**" else: return "Session not found in database." except ValueError as e: return f"⚠️ {e}" else: # Show the current title and session ID title = self._session_db.get_session_title(session_id) if title: return f"📌 Session: `{session_id}`\nTitle: **{title}**" else: return f"📌 Session: `{session_id}`\nNo title set. Usage: `/title My Session Name`" async def _handle_resume_command(self, event: MessageEvent) -> str: """Handle /resume command — switch to a previously-named session.""" if not self._session_db: return "Session database not available." source = event.source session_key = self._session_key_for_source(source) name = event.get_command_args().strip() if not name: # List recent titled sessions for this user/platform try: user_source = source.platform.value if source.platform else None sessions = self._session_db.list_sessions_rich( source=user_source, limit=10 ) titled = [s for s in sessions if s.get("title")] if not titled: return ( "No named sessions found.\n" "Use `/title My Session` to name your current session, " "then `/resume My Session` to return to it later." ) lines = ["📋 **Named Sessions**\n"] for s in titled[:10]: title = s["title"] preview = s.get("preview", "")[:40] preview_part = f" — _{preview}_" if preview else "" lines.append(f"• **{title}**{preview_part}") lines.append("\nUsage: `/resume `") return "\n".join(lines) except Exception as e: logger.debug("Failed to list titled sessions: %s", e) return f"Could not list sessions: {e}" # Resolve the name to a session ID. target_id = self._session_db.resolve_session_by_title(name) if not target_id: return ( f"No session found matching '**{name}**'.\n" "Use `/resume` with no arguments to see available sessions." ) # Compression creates child continuations that hold the live transcript. # Follow that chain so gateway /resume matches CLI behavior (#15000). try: target_id = self._session_db.resolve_resume_session_id(target_id) except Exception as e: logger.debug("Failed to resolve resume continuation for %s: %s", target_id, e) # Check if already on that session current_entry = self.session_store.get_or_create_session(source) if current_entry.session_id == target_id: return f"📌 Already on session **{name}**." # Clear any running agent for this session key self._release_running_agent_state(session_key) # Switch the session entry to point at the old session new_entry = self.session_store.switch_session(session_key, target_id) if not new_entry: return "Failed to switch session." self._clear_session_boundary_security_state(session_key) # Evict any cached agent for this session so the next message # rebuilds with the correct session_id end-to-end — mirrors # /branch and /reset. Without this, the cached AIAgent (and its # memory provider, which cached `_session_id` during initialize()) # keeps writing into the wrong session's record. See #6672. self._evict_cached_agent(session_key) # Get the title for confirmation title = self._session_db.get_session_title(target_id) or name # Count messages for context history = self.session_store.load_transcript(target_id) msg_count = len([m for m in history if m.get("role") == "user"]) if history else 0 msg_part = f" ({msg_count} message{'s' if msg_count != 1 else ''})" if msg_count else "" return f"↻ Resumed session **{title}**{msg_part}. Conversation restored." async def _handle_branch_command(self, event: MessageEvent) -> str: """Handle /branch [name] — fork the current session into a new independent copy. Copies conversation history to a new session so the user can explore a different approach without losing the original. Inspired by Claude Code's /branch command. """ import uuid as _uuid if not self._session_db: return "Session database not available." source = event.source session_key = self._session_key_for_source(source) # Load the current session and its transcript current_entry = self.session_store.get_or_create_session(source) history = self.session_store.load_transcript(current_entry.session_id) if not history: return "No conversation to branch — send a message first." branch_name = event.get_command_args().strip() # Generate the new session ID from datetime import datetime as _dt now = _dt.now() timestamp_str = now.strftime("%Y%m%d_%H%M%S") short_uuid = _uuid.uuid4().hex[:6] new_session_id = f"{timestamp_str}_{short_uuid}" # Determine branch title if branch_name: branch_title = branch_name else: current_title = self._session_db.get_session_title(current_entry.session_id) base = current_title or "branch" branch_title = self._session_db.get_next_title_in_lineage(base) parent_session_id = current_entry.session_id # Create the new session with parent link try: self._session_db.create_session( session_id=new_session_id, source=source.platform.value if source.platform else "gateway", model=(self.config.get("model", {}) or {}).get("default") if isinstance(self.config, dict) else None, parent_session_id=parent_session_id, ) except Exception as e: logger.error("Failed to create branch session: %s", e) return f"Failed to create branch: {e}" # Copy conversation history to the new session for msg in history: try: self._session_db.append_message( session_id=new_session_id, role=msg.get("role", "user"), content=msg.get("content"), tool_name=msg.get("tool_name") or msg.get("name"), tool_calls=msg.get("tool_calls"), tool_call_id=msg.get("tool_call_id"), finish_reason=msg.get("finish_reason"), reasoning=msg.get("reasoning"), reasoning_content=msg.get("reasoning_content"), reasoning_details=msg.get("reasoning_details"), codex_reasoning_items=msg.get("codex_reasoning_items"), codex_message_items=msg.get("codex_message_items"), ) except Exception: pass # Best-effort copy # Set title try: self._session_db.set_session_title(new_session_id, branch_title) except Exception: pass # Switch the session store entry to the new session new_entry = self.session_store.switch_session(session_key, new_session_id) if not new_entry: return "Branch created but failed to switch to it." self._clear_session_boundary_security_state(session_key) # Evict any cached agent for this session self._evict_cached_agent(session_key) msg_count = len([m for m in history if m.get("role") == "user"]) return ( f"⑂ Branched to **{branch_title}**" f" ({msg_count} message{'s' if msg_count != 1 else ''} copied)\n" f"Original: `{parent_session_id}`\n" f"Branch: `{new_session_id}`\n" f"Use `/resume` to switch back to the original." ) async def _handle_usage_command(self, event: MessageEvent) -> str: """Handle /usage command -- show token usage for the current session. Checks both _running_agents (mid-turn) and _agent_cache (between turns) so that rate limits, cost estimates, and detailed token breakdowns are available whenever the user asks, not only while the agent is running. """ source = event.source session_key = self._session_key_for_source(source) # Try running agent first (mid-turn), then cached agent (between turns) agent = self._running_agents.get(session_key) if not agent or agent is _AGENT_PENDING_SENTINEL: _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock and _cache is not None: with _cache_lock: cached = _cache.get(session_key) if cached: agent = cached[0] # Resolve provider/base_url/api_key for the account-usage fetch. # Prefer the live agent; fall back to persisted billing data on the # SessionDB row so `/usage` still returns account info between turns # when no agent is resident. provider = getattr(agent, "provider", None) if agent and agent is not _AGENT_PENDING_SENTINEL else None base_url = getattr(agent, "base_url", None) if agent and agent is not _AGENT_PENDING_SENTINEL else None api_key = getattr(agent, "api_key", None) if agent and agent is not _AGENT_PENDING_SENTINEL else None if not provider and getattr(self, "_session_db", None) is not None: try: _entry_for_billing = self.session_store.get_or_create_session(source) persisted = self._session_db.get_session(_entry_for_billing.session_id) or {} except Exception: persisted = {} provider = provider or persisted.get("billing_provider") base_url = base_url or persisted.get("billing_base_url") # Fetch account usage off the event loop so slow provider APIs don't # block the gateway. Failures are non-fatal -- account_lines stays []. account_lines: list[str] = [] if provider: try: account_snapshot = await asyncio.to_thread( fetch_account_usage, provider, base_url=base_url, api_key=api_key, ) except Exception: account_snapshot = None if account_snapshot: account_lines = render_account_usage_lines(account_snapshot, markdown=True) if agent and hasattr(agent, "session_total_tokens") and agent.session_api_calls > 0: lines = [] # Rate limits (when available from provider headers) rl_state = agent.get_rate_limit_state() if rl_state and rl_state.has_data: from agent.rate_limit_tracker import format_rate_limit_compact lines.append(f"⏱️ **Rate Limits:** {format_rate_limit_compact(rl_state)}") lines.append("") # Session token usage — detailed breakdown matching CLI input_tokens = getattr(agent, "session_input_tokens", 0) or 0 output_tokens = getattr(agent, "session_output_tokens", 0) or 0 cache_read = getattr(agent, "session_cache_read_tokens", 0) or 0 cache_write = getattr(agent, "session_cache_write_tokens", 0) or 0 lines.append("📊 **Session Token Usage**") lines.append(f"Model: `{agent.model}`") lines.append(f"Input tokens: {input_tokens:,}") if cache_read: lines.append(f"Cache read tokens: {cache_read:,}") if cache_write: lines.append(f"Cache write tokens: {cache_write:,}") lines.append(f"Output tokens: {output_tokens:,}") lines.append(f"Total: {agent.session_total_tokens:,}") lines.append(f"API calls: {agent.session_api_calls}") # Cost estimation try: from agent.usage_pricing import CanonicalUsage, estimate_usage_cost cost_result = estimate_usage_cost( agent.model, CanonicalUsage( input_tokens=input_tokens, output_tokens=output_tokens, cache_read_tokens=cache_read, cache_write_tokens=cache_write, ), provider=getattr(agent, "provider", None), base_url=getattr(agent, "base_url", None), ) if cost_result.amount_usd is not None: prefix = "~" if cost_result.status == "estimated" else "" lines.append(f"Cost: {prefix}${float(cost_result.amount_usd):.4f}") elif cost_result.status == "included": lines.append("Cost: included") except Exception: pass # Context window and compressions ctx = agent.context_compressor if ctx.last_prompt_tokens: pct = min(100, ctx.last_prompt_tokens / ctx.context_length * 100) if ctx.context_length else 0 lines.append(f"Context: {ctx.last_prompt_tokens:,} / {ctx.context_length:,} ({pct:.0f}%)") if ctx.compression_count: lines.append(f"Compressions: {ctx.compression_count}") if account_lines: lines.append("") lines.extend(account_lines) return "\n".join(lines) # No agent at all -- check session history for a rough count session_entry = self.session_store.get_or_create_session(source) history = self.session_store.load_transcript(session_entry.session_id) if history: from agent.model_metadata import estimate_messages_tokens_rough msgs = [m for m in history if m.get("role") in ("user", "assistant") and m.get("content")] approx = estimate_messages_tokens_rough(msgs) lines = [ "📊 **Session Info**", f"Messages: {len(msgs)}", f"Estimated context: ~{approx:,} tokens", "_(Detailed usage available after the first agent response)_", ] if account_lines: lines.append("") lines.extend(account_lines) return "\n".join(lines) if account_lines: return "\n".join(account_lines) return "No usage data available for this session." async def _handle_insights_command(self, event: MessageEvent) -> str: """Handle /insights command -- show usage insights and analytics.""" args = event.get_command_args().strip() # Normalize Unicode dashes (Telegram/iOS auto-converts -- to em/en dash) args = re.sub(r'[\u2012\u2013\u2014\u2015](days|source)', r'--\1', args) days = 30 source = None # Parse simple args: /insights 7 or /insights --days 7 if args: parts = args.split() i = 0 while i < len(parts): if parts[i] == "--days" and i + 1 < len(parts): try: days = int(parts[i + 1]) except ValueError: return f"Invalid --days value: {parts[i + 1]}" i += 2 elif parts[i] == "--source" and i + 1 < len(parts): source = parts[i + 1] i += 2 elif parts[i].isdigit(): days = int(parts[i]) i += 1 else: i += 1 try: from hermes_state import SessionDB from agent.insights import InsightsEngine loop = asyncio.get_running_loop() def _run_insights(): db = SessionDB() engine = InsightsEngine(db) report = engine.generate(days=days, source=source) result = engine.format_gateway(report) db.close() return result return await loop.run_in_executor(None, _run_insights) except Exception as e: logger.error("Insights command error: %s", e, exc_info=True) return f"Error generating insights: {e}" async def _handle_reload_mcp_command(self, event: MessageEvent) -> Optional[str]: """Handle /reload-mcp — reconnect MCP servers and rebuild the cached agent. Reloading MCP tools invalidates the provider prompt cache for the active session (tool schemas are baked into the system prompt). The next message re-sends full input tokens, which is expensive on long-context or high-reasoning models. To surface that cost, the command routes through the slash-confirm primitive: users get an Approve Once / Always Approve / Cancel prompt before the reload actually runs. "Always Approve" persists ``approvals.mcp_reload_confirm: false`` so the prompt is silenced for subsequent reloads in any session. Users can also skip the confirm by flipping the config key directly. """ source = event.source session_key = self._session_key_for_source(source) # Read the gate fresh from disk so a prior "always" click takes # effect on the next invocation without restarting the gateway. user_config = self._read_user_config() approvals = user_config.get("approvals") if isinstance(user_config, dict) else None confirm_required = True if isinstance(approvals, dict): confirm_required = bool(approvals.get("mcp_reload_confirm", True)) if not confirm_required: return await self._execute_mcp_reload(event) # Route through slash-confirm. The primitive sends the prompt and # stores the resume handler; the button/text response triggers # ``_resolve_slash_confirm`` which invokes the handler with the # chosen outcome. async def _on_confirm(choice: str) -> Optional[str]: if choice == "cancel": return "🟡 /reload-mcp cancelled. MCP tools unchanged." if choice == "always": # Persist the opt-out and run the reload. try: from cli import save_config_value save_config_value("approvals.mcp_reload_confirm", False) logger.info( "User opted out of /reload-mcp confirmation (session=%s)", session_key, ) except Exception as exc: logger.warning("Failed to persist mcp_reload_confirm=false: %s", exc) # once / always → run the reload result = await self._execute_mcp_reload(event) if choice == "always": return ( f"{result}\n\n" "ℹ️ Future `/reload-mcp` calls will run without confirmation. " "Re-enable via `approvals.mcp_reload_confirm: true` in config.yaml." ) return result prompt_message = ( "⚠️ **Confirm /reload-mcp**\n\n" "Reloading MCP servers rebuilds the tool set for this session " "and **invalidates the provider prompt cache** — the next " "message will re-send full input tokens. On long-context or " "high-reasoning models this can be expensive.\n\n" "Choose:\n" "• **Approve Once** — reload now\n" "• **Always Approve** — reload now and silence this prompt permanently\n" "• **Cancel** — leave MCP tools unchanged\n\n" "_Text fallback: reply `/approve`, `/always`, or `/cancel`._" ) return await self._request_slash_confirm( event=event, command="reload-mcp", title="/reload-mcp", message=prompt_message, handler=_on_confirm, ) async def _execute_mcp_reload(self, event: MessageEvent) -> str: """Actually disconnect, reconnect, and notify MCP tool changes. Split out from ``_handle_reload_mcp_command`` so the confirmation wrapper can invoke the same path whether the user confirmed via button, text reply, or has the confirm gate disabled. """ loop = asyncio.get_running_loop() try: from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock # Capture old server names before shutdown with _lock: old_servers = set(_servers.keys()) # Read new config before shutting down, so we know what will be added/removed # Shutdown existing connections await loop.run_in_executor(None, shutdown_mcp_servers) # Reconnect by discovering tools (reads config.yaml fresh) new_tools = await loop.run_in_executor(None, discover_mcp_tools) # Compute what changed with _lock: connected_servers = set(_servers.keys()) added = connected_servers - old_servers removed = old_servers - connected_servers reconnected = connected_servers & old_servers lines = ["🔄 **MCP Servers Reloaded**\n"] if reconnected: lines.append(f"♻️ Reconnected: {', '.join(sorted(reconnected))}") if added: lines.append(f"➕ Added: {', '.join(sorted(added))}") if removed: lines.append(f"➖ Removed: {', '.join(sorted(removed))}") if not connected_servers: lines.append("No MCP servers connected.") else: lines.append(f"\n🔧 {len(new_tools)} tool(s) available from {len(connected_servers)} server(s)") # Inject a message at the END of the session history so the # model knows tools changed on its next turn. Appended after # all existing messages to preserve prompt-cache for the prefix. change_parts = [] if added: change_parts.append(f"Added servers: {', '.join(sorted(added))}") if removed: change_parts.append(f"Removed servers: {', '.join(sorted(removed))}") if reconnected: change_parts.append(f"Reconnected servers: {', '.join(sorted(reconnected))}") tool_summary = f"{len(new_tools)} MCP tool(s) now available" if new_tools else "No MCP tools available" change_detail = ". ".join(change_parts) + ". " if change_parts else "" reload_msg = { "role": "user", "content": f"[IMPORTANT: MCP servers have been reloaded. {change_detail}{tool_summary}. The tool list for this conversation has been updated accordingly.]", } try: session_entry = self.session_store.get_or_create_session(event.source) self.session_store.append_to_transcript( session_entry.session_id, reload_msg ) except Exception: pass # Best-effort; don't fail the reload over a transcript write return "\n".join(lines) except Exception as e: logger.warning("MCP reload failed: %s", e) return f"❌ MCP reload failed: {e}" async def _handle_reload_skills_command(self, event: MessageEvent) -> str: """Handle /reload-skills — rescan skills dir, queue a note for next turn. Skills don't need to be in the system prompt for the model to use them (they're invoked via ``/skill-name``, ``skills_list``, or ``skill_view`` at runtime), so this does NOT clear the prompt cache — prefix caching stays intact. If any skills were added or removed, a one-shot note is queued on ``self._pending_skills_reload_notes[session_key]``. The gateway prepends it to the NEXT user message in this session (see the consumer at ~L11025 in ``_run_agent_turn``), then clears it. Nothing is written to the session transcript out-of-band, so message alternation is preserved. """ loop = asyncio.get_running_loop() try: from agent.skill_commands import reload_skills result = await loop.run_in_executor(None, reload_skills) added = result.get("added", []) # [{"name", "description"}, ...] removed = result.get("removed", []) # [{"name", "description"}, ...] total = result.get("total", 0) # Let each connected adapter refresh any platform-side state # that cached the skill list at startup. Today that's the # Discord /skill autocomplete (registered once per connect); # without this call, new skills stay invisible in the # dropdown and deleted skills error out when clicked. Other # adapters that don't override refresh_skill_group (Telegram's # BotCommand menu, Slack subcommand map, etc.) are silently # skipped — the in-process reload above is enough for them. for adapter in list(self.adapters.values()): refresh = getattr(adapter, "refresh_skill_group", None) if not callable(refresh): continue try: maybe = refresh() if inspect.isawaitable(maybe): await maybe except Exception as exc: logger.warning( "Adapter %s refresh_skill_group raised: %s", getattr(adapter, "name", adapter), exc, ) lines = ["🔄 **Skills Reloaded**\n"] if not added and not removed: lines.append("No new skills detected.") lines.append(f"\n📚 {total} skill(s) available") return "\n".join(lines) def _fmt_line(item: dict) -> str: nm = item.get("name", "") desc = item.get("description", "") return f" - {nm}: {desc}" if desc else f" - {nm}" if added: lines.append("➕ **Added Skills:**") for item in added: lines.append(_fmt_line(item)) if removed: lines.append("➖ **Removed Skills:**") for item in removed: lines.append(_fmt_line(item)) lines.append(f"\n📚 {total} skill(s) available") # Queue the one-shot note for the next user turn in this session. # Format matches how the system prompt renders pre-existing # skills (`` - name: description``) so the model reads the # diff in the same shape as its original skill catalog. sections = ["[USER INITIATED SKILLS RELOAD:"] if added: sections.append("") sections.append("Added Skills:") for item in added: sections.append(_fmt_line(item)) if removed: sections.append("") sections.append("Removed Skills:") for item in removed: sections.append(_fmt_line(item)) sections.append("") sections.append("Use skills_list to see the updated catalog.]") note = "\n".join(sections) session_key = self._session_key_for_source(event.source) if not hasattr(self, "_pending_skills_reload_notes"): self._pending_skills_reload_notes = {} if session_key: self._pending_skills_reload_notes[session_key] = note return "\n".join(lines) except Exception as e: logger.warning("Skills reload failed: %s", e) return f"❌ Skills reload failed: {e}" # ------------------------------------------------------------------ # Slash-command confirmation primitive (generic) # ------------------------------------------------------------------ # Used by slash commands that have a non-destructive but expensive # side effect worth an explicit user confirmation (currently only # /reload-mcp, which invalidates the prompt cache). Two delivery # paths: # 1. Button UI — adapters that override ``send_slash_confirm`` # (Telegram, Discord, Slack, Matrix, Feishu) render three # inline buttons. The adapter routes the button click back via # ``tools.slash_confirm.resolve(session_key, confirm_id, choice)``. # 2. Text fallback — adapters that don't override the hook get a # plain text prompt. Users reply with /approve, /always, or # /cancel; the early intercept in ``_handle_message`` matches # those replies against ``tools.slash_confirm.get_pending()``. async def _request_slash_confirm( self, *, event: MessageEvent, command: str, title: str, message: str, handler, ) -> Optional[str]: """Ask the user to confirm an expensive slash command. ``handler`` is an async callable ``handler(choice: str) -> str`` where ``choice`` is ``"once"``, ``"always"``, or ``"cancel"``. The handler runs on the event loop when the user responds; its return value is sent back as a gateway message. Returns a short acknowledgment string to send immediately (before the user's response). If buttons rendered successfully the ack is ``None`` (buttons are self-explanatory); if we fell back to text the message itself IS the ack. """ from tools import slash_confirm as _slash_confirm_mod source = event.source session_key = self._session_key_for_source(source) confirm_id = f"{next(self._slash_confirm_counter)}" # Register the pending confirm FIRST so a super-fast button click # cannot race the send_slash_confirm return. _slash_confirm_mod.register(session_key, confirm_id, command, handler) adapter = self.adapters.get(source.platform) metadata = self._thread_metadata_for_source(source) used_buttons = False if adapter is not None: try: button_result = await adapter.send_slash_confirm( chat_id=source.chat_id, title=title, message=message, session_key=session_key, confirm_id=confirm_id, metadata=metadata, ) if button_result and getattr(button_result, "success", False): used_buttons = True except Exception as exc: logger.debug( "send_slash_confirm failed for %s on %s: %s", command, source.platform, exc, ) if used_buttons: # Buttons rendered — no redundant text ack. return None # Text fallback — return the prompt message as the direct reply. return message def _read_user_config(self) -> Dict[str, Any]: """Read the user's raw config.yaml (cached) for gate lookups. Used by slash-confirm gates that must reflect on-disk state changes (e.g. a prior "Always Approve" click) without a gateway restart. """ try: from hermes_cli.config import load_config cfg = load_config() return cfg if isinstance(cfg, dict) else {} except Exception: return {} def _thread_metadata_for_source(self, source) -> Optional[Dict[str, Any]]: """Build the metadata dict platforms need for thread-aware replies.""" thread_id = getattr(source, "thread_id", None) if thread_id is None: return None return {"thread_id": thread_id} # ------------------------------------------------------------------ # /approve & /deny — explicit dangerous-command approval # ------------------------------------------------------------------ _APPROVAL_TIMEOUT_SECONDS = 300 # 5 minutes async def _handle_approve_command(self, event: MessageEvent) -> Optional[str]: """Handle /approve command — unblock waiting agent thread(s). The agent thread(s) are blocked inside tools/approval.py waiting for the user to respond. This handler signals the event so the agent resumes and the terminal_tool executes the command inline — the same flow as the CLI's synchronous input() approval. Supports multiple concurrent approvals (parallel subagents, execute_code). ``/approve`` resolves the oldest pending command; ``/approve all`` resolves every pending command at once. Usage: /approve — approve oldest pending command once /approve all — approve ALL pending commands at once /approve session — approve oldest + remember for session /approve all session — approve all + remember for session /approve always — approve oldest + remember permanently /approve all always — approve all + remember permanently """ source = event.source session_key = self._session_key_for_source(source) from tools.approval import ( resolve_gateway_approval, has_blocking_approval, ) if not has_blocking_approval(session_key): if session_key in self._pending_approvals: self._pending_approvals.pop(session_key) return "⚠️ Approval expired (agent is no longer waiting). Ask the agent to try again." return "No pending command to approve." # Parse args: support "all", "all session", "all always", "session", "always" args = event.get_command_args().strip().lower().split() resolve_all = "all" in args remaining = [a for a in args if a != "all"] if any(a in ("always", "permanent", "permanently") for a in remaining): choice = "always" scope_msg = " (pattern approved permanently)" elif any(a in ("session", "ses") for a in remaining): choice = "session" scope_msg = " (pattern approved for this session)" else: choice = "once" scope_msg = "" count = resolve_gateway_approval(session_key, choice, resolve_all=resolve_all) if not count: return "No pending command to approve." # Resume typing indicator — agent is about to continue processing. _adapter = self.adapters.get(source.platform) if _adapter: _adapter.resume_typing_for_chat(source.chat_id) count_msg = f" ({count} commands)" if count > 1 else "" logger.info("User approved %d dangerous command(s) via /approve%s", count, scope_msg) return f"✅ Command{'s' if count > 1 else ''} approved{scope_msg}{count_msg}. The agent is resuming..." async def _handle_deny_command(self, event: MessageEvent) -> str: """Handle /deny command — reject pending dangerous command(s). Signals blocked agent thread(s) with a 'deny' result so they receive a definitive BLOCKED message, same as the CLI deny flow. ``/deny`` denies the oldest; ``/deny all`` denies everything. """ source = event.source session_key = self._session_key_for_source(source) from tools.approval import ( resolve_gateway_approval, has_blocking_approval, ) if not has_blocking_approval(session_key): if session_key in self._pending_approvals: self._pending_approvals.pop(session_key) return "❌ Command denied (approval was stale)." return "No pending command to deny." args = event.get_command_args().strip().lower() resolve_all = "all" in args count = resolve_gateway_approval(session_key, "deny", resolve_all=resolve_all) if not count: return "No pending command to deny." # Resume typing indicator — agent continues (with BLOCKED result). _adapter = self.adapters.get(source.platform) if _adapter: _adapter.resume_typing_for_chat(source.chat_id) count_msg = f" ({count} commands)" if count > 1 else "" logger.info("User denied %d dangerous command(s) via /deny", count) return f"❌ Command{'s' if count > 1 else ''} denied{count_msg}." # Platforms where /update is allowed. ACP, API server, and webhooks are # programmatic interfaces that should not trigger system updates. _UPDATE_ALLOWED_PLATFORMS = frozenset({ Platform.TELEGRAM, Platform.DISCORD, Platform.SLACK, Platform.WHATSAPP, Platform.SIGNAL, Platform.MATTERMOST, Platform.MATRIX, Platform.HOMEASSISTANT, Platform.EMAIL, Platform.SMS, Platform.DINGTALK, Platform.FEISHU, Platform.WECOM, Platform.WECOM_CALLBACK, Platform.WEIXIN, Platform.BLUEBUBBLES, Platform.QQBOT, Platform.LOCAL, }) async def _handle_debug_command(self, event: MessageEvent) -> str: """Handle /debug — upload debug report (summary only) and return paste URLs. Gateway uploads ONLY the summary report (system info + log tails), NOT full log files, to protect conversation privacy. Users who need full log uploads should use ``hermes debug share`` from the CLI. """ import asyncio from hermes_cli.debug import ( _capture_dump, collect_debug_report, upload_to_pastebin, _schedule_auto_delete, _GATEWAY_PRIVACY_NOTICE, _best_effort_sweep_expired_pastes, ) loop = asyncio.get_running_loop() # Run blocking I/O (dump capture, log reads, uploads) in a thread. def _collect_and_upload(): _best_effort_sweep_expired_pastes() dump_text = _capture_dump() report = collect_debug_report(log_lines=200, dump_text=dump_text) urls = {} try: urls["Report"] = upload_to_pastebin(report) except Exception as exc: return f"✗ Failed to upload debug report: {exc}" # Schedule auto-deletion after 6 hours _schedule_auto_delete(list(urls.values())) lines = [_GATEWAY_PRIVACY_NOTICE, "", "**Debug report uploaded:**", ""] label_width = max(len(k) for k in urls) for label, url in urls.items(): lines.append(f"`{label:<{label_width}}` {url}") lines.append("") lines.append("⏱ Pastes will auto-delete in 6 hours.") lines.append("For full log uploads, use `hermes debug share` from the CLI.") lines.append("Share these links with the Hermes team for support.") return "\n".join(lines) return await loop.run_in_executor(None, _collect_and_upload) async def _handle_update_command(self, event: MessageEvent) -> str: """Handle /update command — update Hermes Agent to the latest version. Spawns ``hermes update`` in a detached session (via ``setsid``) so it survives the gateway restart that ``hermes update`` may trigger. Marker files are written so either the current gateway process or the next one can notify the user when the update finishes. """ import json import shutil import subprocess from datetime import datetime from hermes_cli.config import is_managed, format_managed_message # Block non-messaging platforms (API server, webhooks, ACP) platform = event.source.platform _allowed = self._UPDATE_ALLOWED_PLATFORMS # Plugin platforms with allow_update_command=True are also allowed if platform not in _allowed: try: from gateway.platform_registry import platform_registry entry = platform_registry.get(platform.value) if not entry or not entry.allow_update_command: return "✗ /update is only available from messaging platforms. Run `hermes update` from the terminal." except Exception: return "✗ /update is only available from messaging platforms. Run `hermes update` from the terminal." if is_managed(): return f"✗ {format_managed_message('update Hermes Agent')}" project_root = Path(__file__).parent.parent.resolve() git_dir = project_root / '.git' if not git_dir.exists(): return "✗ Not a git repository — cannot update." hermes_cmd = _resolve_hermes_bin() if not hermes_cmd: return ( "✗ Could not locate the `hermes` command. " "Hermes is running, but the update command could not find the " "executable on PATH or via the current Python interpreter. " "Try running `hermes update` manually in your terminal." ) pending_path = _hermes_home / ".update_pending.json" output_path = _hermes_home / ".update_output.txt" exit_code_path = _hermes_home / ".update_exit_code" session_key = self._session_key_for_source(event.source) pending = { "platform": event.source.platform.value, "chat_id": event.source.chat_id, "user_id": event.source.user_id, "session_key": session_key, "timestamp": datetime.now().isoformat(), } if event.source.thread_id: pending["thread_id"] = event.source.thread_id _tmp_pending = pending_path.with_suffix(".tmp") _tmp_pending.write_text(json.dumps(pending)) _tmp_pending.replace(pending_path) exit_code_path.unlink(missing_ok=True) # Spawn `hermes update --gateway` detached so it survives gateway restart. # --gateway enables file-based IPC for interactive prompts (stash # restore, config migration) so the gateway can forward them to the # user instead of silently skipping them. # Use setsid for portable session detach (works under system services # where systemd-run --user fails due to missing D-Bus session). # PYTHONUNBUFFERED ensures output is flushed line-by-line so the # gateway can stream it to the messenger in near-real-time. hermes_cmd_str = " ".join(shlex.quote(part) for part in hermes_cmd) update_cmd = ( f"PYTHONUNBUFFERED=1 {hermes_cmd_str} update --gateway" f" > {shlex.quote(str(output_path))} 2>&1; " f"status=$?; printf '%s' \"$status\" > {shlex.quote(str(exit_code_path))}" ) try: setsid_bin = shutil.which("setsid") if setsid_bin: # Preferred: setsid creates a new session, fully detached subprocess.Popen( [setsid_bin, "bash", "-c", update_cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) else: # Fallback: start_new_session=True calls os.setsid() in child subprocess.Popen( ["bash", "-c", update_cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) except Exception as e: pending_path.unlink(missing_ok=True) exit_code_path.unlink(missing_ok=True) return f"✗ Failed to start update: {e}" self._schedule_update_notification_watch() return "⚕ Starting Hermes update… I'll stream progress here." def _schedule_update_notification_watch(self) -> None: """Ensure a background task is watching for update completion.""" existing_task = getattr(self, "_update_notification_task", None) if existing_task and not existing_task.done(): return try: self._update_notification_task = asyncio.create_task( self._watch_update_progress() ) except RuntimeError: logger.debug("Skipping update notification watcher: no running event loop") async def _watch_update_progress( self, poll_interval: float = 2.0, stream_interval: float = 4.0, timeout: float = 1800.0, ) -> None: """Watch ``hermes update --gateway``, streaming output + forwarding prompts. Polls ``.update_output.txt`` for new content and sends chunks to the user periodically. Detects ``.update_prompt.json`` (written by the update process when it needs user input) and forwards the prompt to the messenger. The user's next message is intercepted by ``_handle_message`` and written to ``.update_response``. """ pending_path = _hermes_home / ".update_pending.json" claimed_path = _hermes_home / ".update_pending.claimed.json" output_path = _hermes_home / ".update_output.txt" exit_code_path = _hermes_home / ".update_exit_code" prompt_path = _hermes_home / ".update_prompt.json" loop = asyncio.get_running_loop() deadline = loop.time() + timeout # Resolve the adapter and chat_id for sending messages adapter = None chat_id = None session_key = None metadata = None for path in (claimed_path, pending_path): if path.exists(): try: pending = json.loads(path.read_text()) platform_str = pending.get("platform") chat_id = pending.get("chat_id") session_key = pending.get("session_key") thread_id = pending.get("thread_id") metadata = {"thread_id": thread_id} if thread_id else None if platform_str and chat_id: platform = Platform(platform_str) adapter = self.adapters.get(platform) # Fallback session key if not stored (old pending files) if not session_key: session_key = f"{platform_str}:{chat_id}" break except Exception: pass if not adapter or not chat_id: logger.warning("Update watcher: cannot resolve adapter/chat_id, falling back to completion-only") # Fall back to old behavior: wait for exit code and send final notification while (pending_path.exists() or claimed_path.exists()) and loop.time() < deadline: if exit_code_path.exists(): await self._send_update_notification() return await asyncio.sleep(poll_interval) if (pending_path.exists() or claimed_path.exists()) and not exit_code_path.exists(): exit_code_path.write_text("124") await self._send_update_notification() return def _strip_ansi(text: str) -> str: return re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', text) bytes_sent = 0 last_stream_time = loop.time() buffer = "" async def _flush_buffer() -> None: """Send buffered output to the user.""" nonlocal buffer, last_stream_time if not buffer.strip(): buffer = "" return # Chunk to fit message limits (Telegram: 4096, others: generous) clean = _strip_ansi(buffer).strip() buffer = "" last_stream_time = loop.time() if not clean: return # Split into chunks if too long max_chunk = 3500 chunks = [clean[i:i + max_chunk] for i in range(0, len(clean), max_chunk)] for chunk in chunks: try: await adapter.send(chat_id, f"```\n{chunk}\n```", metadata=metadata) except Exception as e: logger.debug("Update stream send failed: %s", e) while loop.time() < deadline: # Check for completion if exit_code_path.exists(): # Read any remaining output if output_path.exists(): try: content = output_path.read_text() if len(content) > bytes_sent: buffer += content[bytes_sent:] bytes_sent = len(content) except OSError: pass await _flush_buffer() # Send final status try: exit_code_raw = exit_code_path.read_text().strip() or "1" exit_code = int(exit_code_raw) if exit_code == 0: await adapter.send(chat_id, "✅ Hermes update finished.", metadata=metadata) else: await adapter.send( chat_id, "❌ Hermes update failed (exit code {}).".format(exit_code), metadata=metadata, ) logger.info("Update finished (exit=%s), notified %s", exit_code, session_key) except Exception as e: logger.warning("Update final notification failed: %s", e) # Cleanup for p in (pending_path, claimed_path, output_path, exit_code_path, prompt_path): p.unlink(missing_ok=True) (_hermes_home / ".update_response").unlink(missing_ok=True) self._update_prompt_pending.pop(session_key, None) return # Check for new output if output_path.exists(): try: content = output_path.read_text() if len(content) > bytes_sent: buffer += content[bytes_sent:] bytes_sent = len(content) except OSError: pass # Flush buffer periodically if buffer.strip() and (loop.time() - last_stream_time) >= stream_interval: await _flush_buffer() # Check for prompts — only forward if we haven't already sent # one that's still awaiting a response. Without this guard the # watcher would re-read the same .update_prompt.json every poll # cycle and spam the user with duplicate prompt messages. if (prompt_path.exists() and session_key and not self._update_prompt_pending.get(session_key)): try: prompt_data = json.loads(prompt_path.read_text()) prompt_text = prompt_data.get("prompt", "") default = prompt_data.get("default", "") if prompt_text: # Flush any buffered output first so the user sees # context before the prompt await _flush_buffer() # Try platform-native buttons first (Discord, Telegram) sent_buttons = False if getattr(type(adapter), "send_update_prompt", None) is not None: try: await adapter.send_update_prompt( chat_id=chat_id, prompt=prompt_text, default=default, session_key=session_key, metadata=metadata, ) sent_buttons = True except Exception as btn_err: logger.debug("Button-based update prompt failed: %s", btn_err) if not sent_buttons: default_hint = f" (default: {default})" if default else "" await adapter.send( chat_id, f"⚕ **Update needs your input:**\n\n" f"{prompt_text}{default_hint}\n\n" f"Reply `/approve` (yes) or `/deny` (no), " f"or type your answer directly.", metadata=metadata, ) self._update_prompt_pending[session_key] = True # Remove the prompt file so it isn't re-read on the # next poll cycle. The update process only needs # .update_response to continue — it doesn't re-check # .update_prompt.json while waiting. prompt_path.unlink(missing_ok=True) logger.info("Forwarded update prompt to %s: %s", session_key, prompt_text[:80]) except (json.JSONDecodeError, OSError) as e: logger.debug("Failed to read update prompt: %s", e) await asyncio.sleep(poll_interval) # Timeout if not exit_code_path.exists(): logger.warning("Update watcher timed out after %.0fs", timeout) exit_code_path.write_text("124") await _flush_buffer() try: await adapter.send( chat_id, "❌ Hermes update timed out after 30 minutes.", metadata=metadata, ) except Exception: pass for p in (pending_path, claimed_path, output_path, exit_code_path, prompt_path): p.unlink(missing_ok=True) (_hermes_home / ".update_response").unlink(missing_ok=True) self._update_prompt_pending.pop(session_key, None) async def _send_update_notification(self) -> bool: """If an update finished, notify the user. Returns False when the update is still running so a caller can retry later. Returns True after a definitive send/skip decision. This is the legacy notification path used when the streaming watcher cannot resolve the adapter (e.g. after a gateway restart where the platform hasn't reconnected yet). """ pending_path = _hermes_home / ".update_pending.json" claimed_path = _hermes_home / ".update_pending.claimed.json" output_path = _hermes_home / ".update_output.txt" exit_code_path = _hermes_home / ".update_exit_code" if not pending_path.exists() and not claimed_path.exists(): return False cleanup = True active_pending_path = claimed_path try: if pending_path.exists(): try: pending_path.replace(claimed_path) except FileNotFoundError: if not claimed_path.exists(): return True elif not claimed_path.exists(): return True pending = json.loads(claimed_path.read_text()) platform_str = pending.get("platform") chat_id = pending.get("chat_id") thread_id = pending.get("thread_id") if not exit_code_path.exists(): logger.info("Update notification deferred: update still running") cleanup = False active_pending_path = pending_path claimed_path.replace(pending_path) return False exit_code_raw = exit_code_path.read_text().strip() or "1" exit_code = int(exit_code_raw) # Read the captured update output output = "" if output_path.exists(): output = output_path.read_text() # Resolve adapter platform = Platform(platform_str) adapter = self.adapters.get(platform) if adapter and chat_id: metadata = {"thread_id": thread_id} if thread_id else None # Strip ANSI escape codes for clean display output = re.sub(r'\x1b\[[0-9;]*m', '', output).strip() if output: if len(output) > 3500: output = "…" + output[-3500:] if exit_code == 0: msg = f"✅ Hermes update finished.\n\n```\n{output}\n```" else: msg = f"❌ Hermes update failed.\n\n```\n{output}\n```" else: if exit_code == 0: msg = "✅ Hermes update finished successfully." else: msg = "❌ Hermes update failed. Check the gateway logs or run `hermes update` manually for details." await adapter.send(chat_id, msg, metadata=metadata) logger.info( "Sent post-update notification to %s:%s (exit=%s)", platform_str, chat_id, exit_code, ) except Exception as e: logger.warning("Post-update notification failed: %s", e) finally: if cleanup: active_pending_path.unlink(missing_ok=True) claimed_path.unlink(missing_ok=True) output_path.unlink(missing_ok=True) exit_code_path.unlink(missing_ok=True) return True async def _send_restart_notification(self) -> None: """Notify the chat that initiated /restart that the gateway is back.""" notify_path = _hermes_home / ".restart_notify.json" if not notify_path.exists(): return try: data = json.loads(notify_path.read_text()) platform_str = data.get("platform") chat_id = data.get("chat_id") thread_id = data.get("thread_id") if not platform_str or not chat_id: return platform = Platform(platform_str) adapter = self.adapters.get(platform) if not adapter: logger.debug( "Restart notification skipped: %s adapter not connected", platform_str, ) return metadata = {"thread_id": thread_id} if thread_id else None result = await adapter.send( chat_id, "♻ Gateway restarted successfully. Your session continues.", metadata=metadata, ) # adapter.send() catches provider errors (e.g. "Chat not found") # and returns SendResult(success=False) rather than raising, so # we must inspect the result before claiming success — otherwise # the log line is misleading and hides real delivery failures. if getattr(result, "success", False): logger.info( "Sent restart notification to %s:%s", platform_str, chat_id, ) else: logger.warning( "Restart notification to %s:%s was not delivered: %s", platform_str, chat_id, getattr(result, "error", "unknown error"), ) except Exception as e: logger.warning("Restart notification failed: %s", e) finally: notify_path.unlink(missing_ok=True) def _set_session_env(self, context: SessionContext) -> list: """Set session context variables for the current async task. Uses ``contextvars`` instead of ``os.environ`` so that concurrent gateway messages cannot overwrite each other's session state. Returns a list of reset tokens; pass them to ``_clear_session_env`` in a ``finally`` block. """ from gateway.session_context import set_session_vars return set_session_vars( platform=context.source.platform.value, chat_id=context.source.chat_id, chat_name=context.source.chat_name or "", thread_id=str(context.source.thread_id) if context.source.thread_id else "", user_id=str(context.source.user_id) if context.source.user_id else "", user_name=str(context.source.user_name) if context.source.user_name else "", session_key=context.session_key, ) def _clear_session_env(self, tokens: list) -> None: """Restore session context variables to their pre-handler values.""" from gateway.session_context import clear_session_vars clear_session_vars(tokens) async def _run_in_executor_with_context(self, func, *args): """Run blocking work in the thread pool while preserving session contextvars.""" loop = asyncio.get_running_loop() ctx = copy_context() return await loop.run_in_executor(None, ctx.run, func, *args) def _decide_image_input_mode(self) -> str: """Resolve the image-input routing for the currently active model. Returns ``"native"`` (attach pixels on the user turn) or ``"text"`` (pre-analyze with vision_analyze and prepend the description). See agent/image_routing.py for the full decision table. The active provider/model are read from config.yaml so the decision tracks ``/model`` switches automatically on the next message. """ try: from agent.image_routing import decide_image_input_mode from agent.auxiliary_client import _read_main_model, _read_main_provider from hermes_cli.config import load_config cfg = load_config() provider = _read_main_provider() model = _read_main_model() return decide_image_input_mode(provider, model, cfg) except Exception as exc: logger.debug("image_routing: decision failed, falling back to text — %s", exc) return "text" async def _enrich_message_with_vision( self, user_text: str, image_paths: List[str], ) -> str: """ Auto-analyze user-attached images with the vision tool and prepend the descriptions to the message text. Each image is analyzed with a general-purpose prompt. The resulting description *and* the local cache path are injected so the model can: 1. Immediately understand what the user sent (no extra tool call). 2. Re-examine the image with vision_analyze if it needs more detail. Args: user_text: The user's original caption / message text. image_paths: List of local file paths to cached images. Returns: The enriched message string with vision descriptions prepended. """ from tools.vision_tools import vision_analyze_tool from agent.memory_manager import sanitize_context analysis_prompt = ( "Describe everything visible in this image in thorough detail. " "Include any text, code, data, objects, people, layout, colors, " "and any other notable visual information." ) enriched_parts = [] for path in image_paths: try: logger.debug("Auto-analyzing user image: %s", path) result_json = await vision_analyze_tool( image_url=path, user_prompt=analysis_prompt, ) result = json.loads(result_json) if result.get("success"): description = result.get("analysis", "") description = sanitize_context(description) enriched_parts.append( f"[The user sent an image~ Here's what I can see:\n{description}]\n" f"[If you need a closer look, use vision_analyze with " f"image_url: {path} ~]" ) else: enriched_parts.append( "[The user sent an image but I couldn't quite see it " "this time (>_<) You can try looking at it yourself " f"with vision_analyze using image_url: {path}]" ) except Exception as e: logger.error("Vision auto-analysis error: %s", e) enriched_parts.append( f"[The user sent an image but something went wrong when I " f"tried to look at it~ You can try examining it yourself " f"with vision_analyze using image_url: {path}]" ) # Combine: vision descriptions first, then the user's original text if enriched_parts: prefix = "\n\n".join(enriched_parts) if user_text: return f"{prefix}\n\n{user_text}" return prefix return user_text async def _enrich_message_with_transcription( self, user_text: str, audio_paths: List[str], ) -> str: """ Auto-transcribe user voice/audio messages using the configured STT provider and prepend the transcript to the message text. Args: user_text: The user's original caption / message text. audio_paths: List of local file paths to cached audio files. Returns: The enriched message string with transcriptions prepended. """ if not getattr(self.config, "stt_enabled", True): disabled_note = "[The user sent voice message(s), but transcription is disabled in config." if self._has_setup_skill(): disabled_note += ( " You have a skill called hermes-agent-setup that can help " "users configure Hermes features including voice, tools, and more." ) disabled_note += "]" if user_text: return f"{disabled_note}\n\n{user_text}" return disabled_note from tools.transcription_tools import transcribe_audio enriched_parts = [] for path in audio_paths: try: logger.debug("Transcribing user voice: %s", path) result = await asyncio.to_thread(transcribe_audio, path) if result["success"]: transcript = result["transcript"] enriched_parts.append( f'[The user sent a voice message~ ' f'Here\'s what they said: "{transcript}"]' ) else: error = result.get("error", "unknown error") if ( "No STT provider" in error or error.startswith("Neither VOICE_TOOLS_OPENAI_KEY nor OPENAI_API_KEY is set") ): _no_stt_note = ( "[The user sent a voice message but I can't listen " "to it right now — no STT provider is configured. " "A direct message has already been sent to the user " "with setup instructions." ) if self._has_setup_skill(): _no_stt_note += ( " You have a skill called hermes-agent-setup " "that can help users configure Hermes features " "including voice, tools, and more." ) _no_stt_note += "]" enriched_parts.append(_no_stt_note) else: enriched_parts.append( "[The user sent a voice message but I had trouble " f"transcribing it~ ({error})]" ) except Exception as e: logger.error("Transcription error: %s", e) enriched_parts.append( "[The user sent a voice message but something went wrong " "when I tried to listen to it~ Let them know!]" ) if enriched_parts: prefix = "\n\n".join(enriched_parts) # Strip the empty-content placeholder from the Discord adapter # when we successfully transcribed the audio — it's redundant. _placeholder = "(The user sent a message with no text content)" if user_text and user_text.strip() == _placeholder: return prefix if user_text: return f"{prefix}\n\n{user_text}" return prefix return user_text def _build_process_event_source(self, evt: dict): """Resolve the canonical source for a synthetic background-process event. Prefer the persisted session-store origin for the event's session key. Falling back to the currently active foreground event is what causes cross-topic bleed, so don't do that. """ from gateway.session import SessionSource session_key = str(evt.get("session_key") or "").strip() derived_platform = "" derived_chat_type = "" derived_chat_id = "" if session_key: try: self.session_store._ensure_loaded() entry = self.session_store._entries.get(session_key) if entry and getattr(entry, "origin", None): return entry.origin except Exception as exc: logger.debug( "Synthetic process-event session-store lookup failed for %s: %s", session_key, exc, ) _parsed = _parse_session_key(session_key) if _parsed: derived_platform = _parsed["platform"] derived_chat_type = _parsed["chat_type"] derived_chat_id = _parsed["chat_id"] platform_name = str(evt.get("platform") or derived_platform or "").strip().lower() chat_type = str(evt.get("chat_type") or derived_chat_type or "").strip().lower() chat_id = str(evt.get("chat_id") or derived_chat_id or "").strip() if not platform_name or not chat_type or not chat_id: return None try: platform = Platform(platform_name) # Reject arbitrary strings that create dynamic pseudo-members. # Built-in platforms are always valid; plugin platforms must be # registered in the platform registry. if platform.value not in _BUILTIN_PLATFORM_VALUES: try: from gateway.platform_registry import platform_registry if not platform_registry.is_registered(platform.value): raise ValueError(platform_name) except Exception: raise ValueError(platform_name) except Exception: logger.warning( "Synthetic process event has invalid platform metadata: %r", platform_name, ) return None return SessionSource( platform=platform, chat_id=chat_id, chat_type=chat_type, thread_id=str(evt.get("thread_id") or "").strip() or None, user_id=str(evt.get("user_id") or "").strip() or None, user_name=str(evt.get("user_name") or "").strip() or None, ) async def _inject_watch_notification(self, synth_text: str, evt: dict) -> None: """Inject a watch-pattern notification as a synthetic message event. Routing must come from the queued watch event itself, not from whatever foreground message happened to be active when the queue was drained. """ source = self._build_process_event_source(evt) if not source: logger.warning( "Dropping watch notification with no routing metadata for process %s", evt.get("session_id", "unknown"), ) return platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform) adapter = None for p, a in self.adapters.items(): if p.value == platform_name: adapter = a break if not adapter: return try: synth_event = MessageEvent( text=synth_text, message_type=MessageType.TEXT, source=source, internal=True, ) logger.info( "Watch pattern notification — injecting for %s chat=%s thread=%s", platform_name, source.chat_id, source.thread_id, ) await adapter.handle_message(synth_event) except Exception as e: logger.error("Watch notification injection error: %s", e) async def _run_process_watcher(self, watcher: dict) -> None: """ Periodically check a background process and push updates to the user. Runs as an asyncio task. Stays silent when nothing changed. Auto-removes when the process exits or is killed. Notification mode (from ``display.background_process_notifications``): - ``all`` — running-output updates + final message - ``result`` — final completion message only - ``error`` — final message only when exit code != 0 - ``off`` — no messages at all """ from tools.process_registry import process_registry session_id = watcher["session_id"] interval = watcher["check_interval"] session_key = watcher.get("session_key", "") platform_name = watcher.get("platform", "") chat_id = watcher.get("chat_id", "") thread_id = watcher.get("thread_id", "") user_id = watcher.get("user_id", "") user_name = watcher.get("user_name", "") agent_notify = watcher.get("notify_on_complete", False) notify_mode = self._load_background_notifications_mode() logger.debug("Process watcher started: %s (every %ss, notify=%s, agent_notify=%s)", session_id, interval, notify_mode, agent_notify) if notify_mode == "off" and not agent_notify: # Still wait for the process to exit so we can log it, but don't # push any messages to the user. while True: await asyncio.sleep(interval) session = process_registry.get(session_id) if session is None or session.exited: break logger.debug("Process watcher ended (silent): %s", session_id) return last_output_len = 0 while True: await asyncio.sleep(interval) session = process_registry.get(session_id) if session is None: break current_output_len = len(session.output_buffer) has_new_output = current_output_len > last_output_len last_output_len = current_output_len if session.exited: # --- Agent-triggered completion: inject synthetic message --- # Skip if the agent already consumed the result via wait/poll/log from tools.process_registry import process_registry as _pr_check if agent_notify and not _pr_check.is_completion_consumed(session_id): from tools.ansi_strip import strip_ansi _out = strip_ansi(session.output_buffer[-2000:]) if session.output_buffer else "" synth_text = ( f"[IMPORTANT: Background process {session_id} completed " f"(exit code {session.exit_code}).\n" f"Command: {session.command}\n" f"Output:\n{_out}]" ) source = self._build_process_event_source({ "session_id": session_id, "session_key": session_key, "platform": platform_name, "chat_id": chat_id, "thread_id": thread_id, "user_id": user_id, "user_name": user_name, }) if not source: logger.warning( "Dropping completion notification with no routing metadata for process %s", session_id, ) break adapter = None for p, a in self.adapters.items(): if p == source.platform: adapter = a break if adapter and source.chat_id: try: synth_event = MessageEvent( text=synth_text, message_type=MessageType.TEXT, source=source, internal=True, ) logger.info( "Process %s finished — injecting agent notification for session %s chat=%s thread=%s", session_id, session_key, source.chat_id, source.thread_id, ) await adapter.handle_message(synth_event) except Exception as e: logger.error("Agent notify injection error: %s", e) break # --- Normal text-only notification --- # Decide whether to notify based on mode should_notify = ( notify_mode in ("all", "result") or (notify_mode == "error" and session.exit_code not in (0, None)) ) if should_notify: new_output = session.output_buffer[-1000:] if session.output_buffer else "" message_text = ( f"[Background process {session_id} finished with exit code {session.exit_code}~ " f"Here's the final output:\n{new_output}]" ) adapter = None for p, a in self.adapters.items(): if p.value == platform_name: adapter = a break if adapter and chat_id: try: send_meta = {"thread_id": thread_id} if thread_id else None await adapter.send(chat_id, message_text, metadata=send_meta) except Exception as e: logger.error("Watcher delivery error: %s", e) break elif has_new_output and notify_mode == "all" and not agent_notify: # New output available -- deliver status update (only in "all" mode) # Skip periodic updates for agent_notify watchers (they only care about completion) new_output = session.output_buffer[-500:] if session.output_buffer else "" message_text = ( f"[Background process {session_id} is still running~ " f"New output:\n{new_output}]" ) adapter = None for p, a in self.adapters.items(): if p.value == platform_name: adapter = a break if adapter and chat_id: try: send_meta = {"thread_id": thread_id} if thread_id else None await adapter.send(chat_id, message_text, metadata=send_meta) except Exception as e: logger.error("Watcher delivery error: %s", e) logger.debug("Process watcher ended: %s", session_id) _MAX_INTERRUPT_DEPTH = 3 # Cap recursive interrupt handling (#816) # Config keys whose values MUST invalidate the gateway's cached agent # when they change. The agent bakes these into its compressor / context # handling at construction time, so a mid-running-gateway config edit # would otherwise be silently ignored until the user triggers a # different cache eviction (model switch, /reset, etc.). # # Each entry is a tuple of (section, key) read from the raw config dict. # Add more here as new baked-at-construction config settings are added. _CACHE_BUSTING_CONFIG_KEYS: tuple = ( ("model", "context_length"), ("compression", "enabled"), ("compression", "threshold"), ("compression", "target_ratio"), ("compression", "protect_last_n"), ("agent", "disabled_toolsets"), ) @classmethod def _extract_cache_busting_config(cls, user_config: dict | None) -> dict: """Pull values that must bust the cached agent. Returns a flat dict keyed by 'section.key'. Missing config keys and non-dict sections yield None values, which still contribute to the signature (so 'absent' vs 'present-and-null' differ). The live tool registry generation is included too. MCP reloads and dynamic MCP tool-list changes mutate the registry without necessarily changing config.yaml. Cached AIAgent instances freeze their tool schemas at construction time, so a registry generation change must rebuild the agent before the next turn. """ out: Dict[str, Any] = {} cfg = user_config if isinstance(user_config, dict) else {} for section, key in cls._CACHE_BUSTING_CONFIG_KEYS: section_val = cfg.get(section) if isinstance(section_val, dict): out[f"{section}.{key}"] = section_val.get(key) else: out[f"{section}.{key}"] = None try: from tools.registry import registry out["tools.registry_generation"] = getattr(registry, "_generation", None) except Exception: out["tools.registry_generation"] = None return out @staticmethod def _agent_config_signature( model: str, runtime: dict, enabled_toolsets: list, ephemeral_prompt: str, cache_keys: dict | None = None, ) -> str: """Compute a stable string key from agent config values. When this signature changes between messages, the cached AIAgent is discarded and rebuilt. When it stays the same, the cached agent is reused — preserving the frozen system prompt and tool schemas for prompt cache hits. ``cache_keys`` is an optional flat dict of additional config values that should invalidate the cache when they change. Callers pass the output of ``_extract_cache_busting_config(user_config)`` so edits to model.context_length / compression.* in config.yaml are picked up on the next gateway message without a manual restart. """ import hashlib, json as _j # Fingerprint the FULL credential string instead of using a short # prefix. OAuth/JWT-style tokens frequently share a common prefix # (e.g. "eyJhbGci"), which can cause false cache hits across auth # switches if only the first few characters are considered. _api_key = str(runtime.get("api_key", "") or "") _api_key_fingerprint = hashlib.sha256(_api_key.encode()).hexdigest() if _api_key else "" _cache_keys_sorted = sorted((cache_keys or {}).items()) blob = _j.dumps( [ model, _api_key_fingerprint, runtime.get("base_url", ""), runtime.get("provider", ""), runtime.get("api_mode", ""), sorted(enabled_toolsets) if enabled_toolsets else [], # reasoning_config excluded — it's set per-message on the # cached agent and doesn't affect system prompt or tools. ephemeral_prompt or "", _cache_keys_sorted, ], sort_keys=True, default=str, ) return hashlib.sha256(blob.encode()).hexdigest()[:16] def _apply_session_model_override( self, session_key: str, model: str, runtime_kwargs: dict ) -> tuple: """Apply /model session overrides if present, returning (model, runtime_kwargs). The gateway /model command stores per-session overrides in ``_session_model_overrides``. These must take precedence over config.yaml defaults so the switched model is actually used for subsequent messages. Fields with ``None`` values are skipped so partial overrides don't clobber valid config defaults. """ override = self._session_model_overrides.get(session_key) if not override: return model, runtime_kwargs model = override.get("model", model) for key in ("provider", "api_key", "base_url", "api_mode"): val = override.get(key) if val is not None: runtime_kwargs[key] = val return model, runtime_kwargs def _is_intentional_model_switch(self, session_key: str, agent_model: str) -> bool: """Return True if *agent_model* matches an active /model session override.""" override = self._session_model_overrides.get(session_key) return override is not None and override.get("model") == agent_model def _release_running_agent_state( self, session_key: str, *, run_generation: Optional[int] = None, ) -> bool: """Pop ALL per-running-agent state entries for ``session_key``. Replaces ad-hoc ``del self._running_agents[key]`` calls scattered across the gateway. Those sites had drifted: some popped only ``_running_agents``; some also ``_running_agents_ts``; only one path also cleared ``_busy_ack_ts``. Each missed entry was a small, persistent leak — a (str_key → float) tuple per session per gateway lifetime. Use this at every site that ends a running turn, regardless of cause (normal completion, /stop, /reset, /resume, sentinel cleanup, stale-eviction). Per-session state that PERSISTS across turns (``_session_model_overrides``, ``_voice_mode``, ``_pending_approvals``, ``_update_prompt_pending``) is NOT touched here — those have their own lifecycles. When ``run_generation`` is provided, only clear the slot if that generation is still current for the session. This prevents an older async run whose generation was bumped by /stop or /new from clobbering a newer run's state during its own unwind. Returns True when the slot was cleared, False when an ownership guard blocked it. """ if not session_key: return False if run_generation is not None and not self._is_session_run_current( session_key, run_generation ): return False self._running_agents.pop(session_key, None) self._running_agents_ts.pop(session_key, None) if hasattr(self, "_busy_ack_ts"): self._busy_ack_ts.pop(session_key, None) return True def _clear_session_boundary_security_state(self, session_key: str) -> None: """Clear per-session control state that must not survive a boundary switch.""" if not session_key: return pending_approvals = getattr(self, "_pending_approvals", None) if isinstance(pending_approvals, dict): pending_approvals.pop(session_key, None) update_prompt_pending = getattr(self, "_update_prompt_pending", None) if isinstance(update_prompt_pending, dict): update_prompt_pending.pop(session_key, None) try: from tools.approval import clear_session as _clear_approval_session except Exception: return try: _clear_approval_session(session_key) except Exception as e: logger.debug( "Failed to clear approval state for session boundary %s: %s", session_key, e, ) def _begin_session_run_generation(self, session_key: str) -> int: """Claim a fresh run generation token for ``session_key``. Every top-level gateway turn gets a monotonically increasing token. If a later command like /stop or /new invalidates that token while the old worker is still unwinding, the late result can be recognized and dropped instead of bleeding into the fresh session. """ if not session_key: return 0 generations = self.__dict__.get("_session_run_generation") if generations is None: generations = {} self._session_run_generation = generations next_generation = int(generations.get(session_key, 0)) + 1 generations[session_key] = next_generation return next_generation def _invalidate_session_run_generation(self, session_key: str, *, reason: str = "") -> int: """Invalidate any in-flight run token for ``session_key``.""" generation = self._begin_session_run_generation(session_key) if reason: logger.info( "Invalidated run generation for %s → %d (%s)", session_key, generation, reason, ) return generation def _is_session_run_current(self, session_key: str, generation: int) -> bool: """Return True when ``generation`` is still current for ``session_key``.""" if not session_key: return True generations = self.__dict__.get("_session_run_generation") or {} return int(generations.get(session_key, 0)) == int(generation) def _bind_adapter_run_generation( self, adapter: Any, session_key: str, generation: int | None, ) -> None: """Bind a gateway run generation to the adapter's active-session event.""" if not adapter or not session_key or generation is None: return try: interrupt_event = getattr(adapter, "_active_sessions", {}).get(session_key) if interrupt_event is not None: setattr(interrupt_event, "_hermes_run_generation", int(generation)) except Exception: pass async def _interrupt_and_clear_session( self, session_key: str, source: SessionSource, *, interrupt_reason: str, invalidation_reason: str, release_running_state: bool = True, ) -> None: """Interrupt the current run and clear queued session state consistently.""" if not session_key: return running_agent = self._running_agents.get(session_key) if running_agent and running_agent is not _AGENT_PENDING_SENTINEL: running_agent.interrupt(interrupt_reason) self._invalidate_session_run_generation(session_key, reason=invalidation_reason) adapter = self.adapters.get(source.platform) if adapter and hasattr(adapter, "interrupt_session_activity"): await adapter.interrupt_session_activity(session_key, source.chat_id) if adapter and hasattr(adapter, "get_pending_message"): adapter.get_pending_message(session_key) # consume and discard self._pending_messages.pop(session_key, None) if release_running_state: self._release_running_agent_state(session_key) def _evict_cached_agent(self, session_key: str) -> None: """Remove a cached agent for a session (called on /new, /model, etc).""" _lock = getattr(self, "_agent_cache_lock", None) if _lock: with _lock: self._agent_cache.pop(session_key, None) @staticmethod def _init_cached_agent_for_turn(agent: Any, interrupt_depth: int) -> None: """Reset per-turn state on a cached agent before a new turn starts. Both _last_activity_ts and _last_activity_desc are only reset for fresh external turns (depth 0); they are semantically paired — desc describes the activity *at* ts, so updating one without the other would make get_activity_summary() misleading. For interrupt-recursive turns both are preserved so the inactivity watchdog can accumulate stuck-turn idle time and fire the 30-min timeout (#15654). The depth-0 reset is still needed: a session idle for 29 min would otherwise trip the watchdog before the new turn makes its first API call (#9051). """ if interrupt_depth == 0: agent._last_activity_ts = time.time() agent._last_activity_desc = "starting new turn (cached)" agent._api_call_count = 0 def _release_evicted_agent_soft(self, agent: Any) -> None: """Soft cleanup for cache-evicted agents — preserves session tool state. Called from _enforce_agent_cache_cap and _sweep_idle_cached_agents. Distinct from _cleanup_agent_resources (full teardown) because a cache-evicted session may resume at any time — its terminal sandbox, browser daemon, and tracked bg processes must outlive the Python AIAgent instance so the next agent built for the same task_id inherits them. """ if agent is None: return try: if hasattr(agent, "release_clients"): agent.release_clients() else: # Older agent instance (shouldn't happen in practice) — # fall back to the legacy full-close path. self._cleanup_agent_resources(agent) except Exception: pass def _enforce_agent_cache_cap(self) -> None: """Evict oldest cached agents when cache exceeds _AGENT_CACHE_MAX_SIZE. Must be called with _agent_cache_lock held. Resource cleanup (memory provider shutdown, tool resource close) is scheduled on a daemon thread so the caller doesn't block on slow teardown while holding the cache lock. Agents currently in _running_agents are SKIPPED — their clients, terminal sandboxes, background processes, and child subagents are all in active use by the running turn. Evicting them would tear down those resources mid-turn and crash the request. If every candidate in the LRU order is active, we simply leave the cache over the cap; it will be re-checked on the next insert. """ _cache = getattr(self, "_agent_cache", None) if _cache is None: return # OrderedDict.popitem(last=False) pops oldest; plain dict lacks the # arg so skip enforcement if a test fixture swapped the cache type. if not hasattr(_cache, "move_to_end"): return # Snapshot of agent instances that are actively mid-turn. Use id() # so the lookup is O(1) and doesn't depend on AIAgent.__eq__ (which # MagicMock overrides in tests). running_ids = { id(a) for a in getattr(self, "_running_agents", {}).values() if a is not None and a is not _AGENT_PENDING_SENTINEL } # Walk LRU → MRU and evict excess-LRU entries that aren't mid-turn. # We only consider entries in the first (size - cap) LRU positions # as eviction candidates. If one of those slots is held by an # active agent, we SKIP it without compensating by evicting a # newer entry — that would penalise a freshly-inserted session # (which has no cache history to retain) while protecting an # already-cached long-running one. The cache may therefore stay # temporarily over cap; it will re-check on the next insert, # after active turns have finished. excess = max(0, len(_cache) - _AGENT_CACHE_MAX_SIZE) evict_plan: List[tuple] = [] # [(key, agent), ...] if excess > 0: ordered_keys = list(_cache.keys()) for key in ordered_keys[:excess]: entry = _cache.get(key) agent = entry[0] if isinstance(entry, tuple) and entry else None if agent is not None and id(agent) in running_ids: continue # active mid-turn; don't evict, don't substitute evict_plan.append((key, agent)) for key, _ in evict_plan: _cache.pop(key, None) remaining_over_cap = len(_cache) - _AGENT_CACHE_MAX_SIZE if remaining_over_cap > 0: logger.warning( "Agent cache over cap (%d > %d); %d excess slot(s) held by " "mid-turn agents — will re-check on next insert.", len(_cache), _AGENT_CACHE_MAX_SIZE, remaining_over_cap, ) for key, agent in evict_plan: logger.info( "Agent cache at cap; evicting LRU session=%s (cache_size=%d)", key, len(_cache), ) if agent is not None: threading.Thread( target=self._release_evicted_agent_soft, args=(agent,), daemon=True, name=f"agent-cache-evict-{key[:24]}", ).start() def _sweep_idle_cached_agents(self) -> int: """Evict cached agents whose AIAgent has been idle > _AGENT_CACHE_IDLE_TTL_SECS. Safe to call from the session expiry watcher without holding the cache lock — acquires it internally. Returns the number of entries evicted. Resource cleanup is scheduled on daemon threads. Agents currently in _running_agents are SKIPPED for the same reason as _enforce_agent_cache_cap: tearing down an active turn's clients mid-flight would crash the request. """ _cache = getattr(self, "_agent_cache", None) _lock = getattr(self, "_agent_cache_lock", None) if _cache is None or _lock is None: return 0 now = time.time() to_evict: List[tuple] = [] running_ids = { id(a) for a in getattr(self, "_running_agents", {}).values() if a is not None and a is not _AGENT_PENDING_SENTINEL } with _lock: for key, entry in list(_cache.items()): agent = entry[0] if isinstance(entry, tuple) and entry else None if agent is None: continue if id(agent) in running_ids: continue # mid-turn — don't tear it down last_activity = getattr(agent, "_last_activity_ts", None) if last_activity is None: continue if (now - last_activity) > _AGENT_CACHE_IDLE_TTL_SECS: to_evict.append((key, agent)) for key, _ in to_evict: _cache.pop(key, None) for key, agent in to_evict: logger.info( "Agent cache idle-TTL evict: session=%s (idle=%.0fs)", key, now - getattr(agent, "_last_activity_ts", now), ) threading.Thread( target=self._release_evicted_agent_soft, args=(agent,), daemon=True, name=f"agent-cache-idle-{key[:24]}", ).start() return len(to_evict) # ------------------------------------------------------------------ # Proxy mode: forward messages to a remote Hermes API server # ------------------------------------------------------------------ def _get_proxy_url(self) -> Optional[str]: """Return the proxy URL if proxy mode is configured, else None. Checks GATEWAY_PROXY_URL env var first (convenient for Docker), then ``gateway.proxy_url`` in config.yaml. """ url = os.getenv("GATEWAY_PROXY_URL", "").strip() if url: return url.rstrip("/") cfg = _load_gateway_config() url = (cfg.get("gateway") or {}).get("proxy_url", "").strip() if url: return url.rstrip("/") return None async def _run_agent_via_proxy( self, message: str, context_prompt: str, history: List[Dict[str, Any]], source: "SessionSource", session_id: str, session_key: str = None, run_generation: Optional[int] = None, event_message_id: Optional[str] = None, ) -> Dict[str, Any]: """Forward the message to a remote Hermes API server instead of running a local AIAgent. When ``GATEWAY_PROXY_URL`` (or ``gateway.proxy_url`` in config.yaml) is set, the gateway becomes a thin relay: it handles platform I/O (encryption, threading, media) and delegates all agent work to the remote server via ``POST /v1/chat/completions`` with SSE streaming. This lets a Docker container handle Matrix E2EE while the actual agent runs on the host with full access to local files, memory, skills, and a unified session store. """ try: from aiohttp import ClientSession as _AioClientSession, ClientTimeout except ImportError: return { "final_response": "⚠️ Proxy mode requires aiohttp. Install with: pip install aiohttp", "messages": [], "api_calls": 0, "tools": [], } proxy_url = self._get_proxy_url() if not proxy_url: return { "final_response": "⚠️ Proxy URL not configured (GATEWAY_PROXY_URL or gateway.proxy_url)", "messages": [], "api_calls": 0, "tools": [], } proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() def _run_still_current() -> bool: if run_generation is None or not session_key: return True return self._is_session_run_current(session_key, run_generation) # Build messages in OpenAI chat format -------------------------- # # The remote api_server can maintain session continuity via # X-Hermes-Session-Id, so it loads its own history. We only # need to send the current user message. If the remote has # no history for this session yet, include what we have locally # so the first exchange has context. # # We always include the current message. For history, send a # compact version (text-only user/assistant turns) — the remote # handles tool replay and system prompts. api_messages: List[Dict[str, str]] = [] if context_prompt: api_messages.append({"role": "system", "content": context_prompt}) for msg in history: role = msg.get("role") content = msg.get("content") if role in ("user", "assistant") and content: api_messages.append({"role": role, "content": content}) api_messages.append({"role": "user", "content": message}) # HTTP headers --------------------------------------------------- headers: Dict[str, str] = {"Content-Type": "application/json"} if proxy_key: headers["Authorization"] = f"Bearer {proxy_key}" if session_id: headers["X-Hermes-Session-Id"] = session_id body = { "model": "hermes-agent", "messages": api_messages, "stream": True, } # Set up platform streaming if available ------------------------- _stream_consumer = None _scfg = getattr(getattr(self, "config", None), "streaming", None) if _scfg is None: from gateway.config import StreamingConfig _scfg = StreamingConfig() platform_key = _platform_config_key(source.platform) user_config = _load_gateway_config() from gateway.display_config import resolve_display_setting _plat_streaming = resolve_display_setting( user_config, platform_key, "streaming" ) _streaming_enabled = ( _scfg.enabled and _scfg.transport != "off" if _plat_streaming is None else bool(_plat_streaming) ) if source.thread_id: _thread_metadata: Optional[Dict[str, Any]] = {"thread_id": source.thread_id} else: _thread_metadata = None if _streaming_enabled: try: from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig _adapter = self.adapters.get(source.platform) if _adapter: _adapter_supports_edit = getattr(_adapter, "SUPPORTS_MESSAGE_EDITING", True) _effective_cursor = _scfg.cursor if _adapter_supports_edit else "" _buffer_only = False if source.platform == Platform.MATRIX: _effective_cursor = "" _buffer_only = True # Fresh-final applies to Telegram only — other # platforms either edit in place cheaply (Discord, # Slack) or don't have the timestamp-on-edit # problem. (Ported from openclaw/openclaw#72038.) _fresh_final_secs = ( float(getattr(_scfg, "fresh_final_after_seconds", 0.0) or 0.0) if source.platform == Platform.TELEGRAM else 0.0 ) _consumer_cfg = StreamConsumerConfig( edit_interval=_scfg.edit_interval, buffer_threshold=_scfg.buffer_threshold, cursor=_effective_cursor, buffer_only=_buffer_only, fresh_final_after_seconds=_fresh_final_secs, ) _stream_consumer = GatewayStreamConsumer( adapter=_adapter, chat_id=source.chat_id, config=_consumer_cfg, metadata=_thread_metadata, ) except Exception as _sc_err: logger.debug("Proxy: could not set up stream consumer: %s", _sc_err) # Run the stream consumer task in the background stream_task = None if _stream_consumer: stream_task = asyncio.create_task(_stream_consumer.run()) # Send typing indicator _adapter = self.adapters.get(source.platform) if _adapter: try: await _adapter.send_typing(source.chat_id, metadata=_thread_metadata) except Exception: pass # Make the HTTP request with SSE streaming ----------------------- full_response = "" _start = time.time() try: _timeout = ClientTimeout(total=0, sock_read=1800) async with _AioClientSession(timeout=_timeout) as session: async with session.post( f"{proxy_url}/v1/chat/completions", json=body, headers=headers, ) as resp: if resp.status != 200: error_text = await resp.text() logger.warning( "Proxy error (%d) from %s: %s", resp.status, proxy_url, error_text[:500], ) return { "final_response": f"⚠️ Proxy error ({resp.status}): {error_text[:300]}", "messages": [], "api_calls": 0, "tools": [], } # Parse SSE stream buffer = "" async for chunk in resp.content.iter_any(): if not _run_still_current(): logger.info( "Discarding stale proxy stream for %s — generation %d is no longer current", session_key or "?", run_generation or 0, ) return { "final_response": "", "messages": [], "api_calls": 0, "tools": [], "history_offset": len(history), "session_id": session_id, "response_previewed": False, } text = chunk.decode("utf-8", errors="replace") buffer += text # Process complete SSE lines while "\n" in buffer: line, buffer = buffer.split("\n", 1) line = line.strip() if not line: continue if line.startswith("data: "): data = line[6:] if data.strip() == "[DONE]": break try: obj = json.loads(data) choices = obj.get("choices", []) if choices: delta = choices[0].get("delta", {}) content = delta.get("content", "") if content: full_response += content if _stream_consumer: _stream_consumer.on_delta(content) except json.JSONDecodeError: pass except asyncio.CancelledError: raise except Exception as e: logger.error("Proxy connection error to %s: %s", proxy_url, e) if not full_response: return { "final_response": f"⚠️ Proxy connection error: {e}", "messages": [], "api_calls": 0, "tools": [], } # Partial response — return what we got finally: # Finalize stream consumer if _stream_consumer: _stream_consumer.finish() if stream_task: try: await asyncio.wait_for(stream_task, timeout=5.0) except (asyncio.TimeoutError, asyncio.CancelledError): stream_task.cancel() _elapsed = time.time() - _start if not _run_still_current(): logger.info( "Discarding stale proxy result for %s — generation %d is no longer current", session_key or "?", run_generation or 0, ) return { "final_response": "", "messages": [], "api_calls": 0, "tools": [], "history_offset": len(history), "session_id": session_id, "response_previewed": False, } logger.info( "proxy response: url=%s session=%s time=%.1fs response=%d chars", proxy_url, (session_id or "")[:20], _elapsed, len(full_response), ) return { "final_response": full_response or "(No response from remote agent)", "messages": [ {"role": "user", "content": message}, {"role": "assistant", "content": full_response}, ], "api_calls": 1, "tools": [], "history_offset": len(history), "session_id": session_id, "response_previewed": _stream_consumer is not None and bool(full_response), } # ------------------------------------------------------------------ async def _run_agent( self, message: str, context_prompt: str, history: List[Dict[str, Any]], source: SessionSource, session_id: str, session_key: str = None, run_generation: Optional[int] = None, _interrupt_depth: int = 0, event_message_id: Optional[str] = None, channel_prompt: Optional[str] = None, ) -> Dict[str, Any]: """ Run the agent with the given message and context. Returns the full result dict from run_conversation, including: - "final_response": str (the text to send back) - "messages": list (full conversation including tool calls) - "api_calls": int - "completed": bool This is run in a thread pool to not block the event loop. Supports interruption via new messages. """ # ---- Proxy mode: delegate to remote API server ---- if self._get_proxy_url(): return await self._run_agent_via_proxy( message=message, context_prompt=context_prompt, history=history, source=source, session_id=session_id, session_key=session_key, run_generation=run_generation, event_message_id=event_message_id, ) from run_agent import AIAgent import queue def _run_still_current() -> bool: if run_generation is None or not session_key: return True return self._is_session_run_current(session_key, run_generation) user_config = _load_gateway_config() platform_key = _platform_config_key(source.platform) from hermes_cli.tools_config import _get_platform_tools enabled_toolsets = sorted(_get_platform_tools(user_config, platform_key)) agent_cfg_local = user_config.get("agent") or {} disabled_toolsets = agent_cfg_local.get("disabled_toolsets") or None display_config = user_config.get("display", {}) if not isinstance(display_config, dict): display_config = {} # Per-platform display settings — resolve via display_config module # which checks display.platforms.. first, then # display. global, then built-in platform defaults. from gateway.display_config import resolve_display_setting # Apply tool preview length config (0 = no limit) try: from agent.display import set_tool_preview_max_len _tpl = resolve_display_setting(user_config, platform_key, "tool_preview_length", 0) set_tool_preview_max_len(int(_tpl) if _tpl else 0) except Exception: pass # Tool progress mode — resolved per-platform with env var fallback _resolved_tp = resolve_display_setting(user_config, platform_key, "tool_progress") _env_tp = os.getenv("HERMES_TOOL_PROGRESS_MODE") _display_cfg = display_config if isinstance(display_config, dict) else {} _platforms_cfg = _display_cfg.get("platforms") or {} _platform_cfg = _platforms_cfg.get(platform_key) or {} _legacy_tp_overrides = _display_cfg.get("tool_progress_overrides") or {} _tool_progress_configured = ( "tool_progress" in _display_cfg or ( isinstance(_platform_cfg, dict) and "tool_progress" in _platform_cfg ) or ( isinstance(_legacy_tp_overrides, dict) and platform_key in _legacy_tp_overrides ) ) progress_mode = ( _env_tp if _env_tp and not _tool_progress_configured else (_resolved_tp or _env_tp or "all") ) # Disable tool progress for webhooks - they don't support message editing, # so each progress line would be sent as a separate message. from gateway.config import Platform tool_progress_enabled = progress_mode != "off" and source.platform != Platform.WEBHOOK # Natural assistant status messages are intentionally independent from # tool progress and token streaming. Users can keep tool_progress quiet # in chat platforms while opting into concise mid-turn updates. interim_assistant_messages_enabled = ( source.platform != Platform.WEBHOOK and is_truthy_value( display_config.get("interim_assistant_messages"), default=True, ) ) # Queue for progress messages (thread-safe) progress_queue = queue.Queue() if tool_progress_enabled else None last_tool = [None] # Mutable container for tracking in closure last_progress_msg = [None] # Track last message for dedup repeat_count = [0] # How many times the same message repeated # First-touch onboarding latch: fires at most once per run, even if # several tools exceed the threshold. long_tool_hint_fired = [False] _LONG_TOOL_THRESHOLD_S = 30.0 def progress_callback(event_type: str, tool_name: str = None, preview: str = None, args: dict = None, **kwargs): """Callback invoked by agent on tool lifecycle events.""" if not progress_queue or not _run_still_current(): return # First-touch onboarding: the first time a tool takes longer than # _LONG_TOOL_THRESHOLD_S during a run that's streaming every tool # (progress_mode == "all"), append a one-time hint suggesting # /verbose. We only fire when (a) the user hasn't seen the hint # before and (b) /verbose is actually usable on this platform # (gateway gate must be open). The CLI has its own trigger. if event_type == "tool.completed" and not long_tool_hint_fired[0]: try: duration = kwargs.get("duration") or 0 if duration >= _LONG_TOOL_THRESHOLD_S and progress_mode == "all": from agent.onboarding import ( TOOL_PROGRESS_FLAG, is_seen, mark_seen, tool_progress_hint_gateway, ) _cfg = _load_gateway_config() gate_on = is_truthy_value( cfg_get(_cfg, "display", "tool_progress_command"), default=False, ) if gate_on and not is_seen(_cfg, TOOL_PROGRESS_FLAG): long_tool_hint_fired[0] = True progress_queue.put(tool_progress_hint_gateway()) mark_seen(_hermes_home / "config.yaml", TOOL_PROGRESS_FLAG) except Exception as _hint_err: logger.debug("tool-progress onboarding hint failed: %s", _hint_err) return # Only act on tool.started events (ignore tool.completed, reasoning.available, etc.) if event_type not in ("tool.started",): return # Suppress tool-progress bubbles once the user has sent `stop`. # When the LLM response carries N parallel tool calls, the agent # fires N "tool.started" events back-to-back before checking for # interrupts — without this guard, a late `stop` still renders # all N as 🔍 bubbles, making the interrupt feel ignored. # (agent lives in run_sync's scope; agent_holder[0] is the shared # handle across nested scopes — see line ~9607.) try: _agent_for_interrupt = agent_holder[0] if agent_holder else None if _agent_for_interrupt is not None and getattr( _agent_for_interrupt, "is_interrupted", False ): return except Exception: pass # "new" mode: only report when tool changes if progress_mode == "new" and tool_name == last_tool[0]: return last_tool[0] = tool_name # Build progress message with primary argument preview from agent.display import get_tool_emoji emoji = get_tool_emoji(tool_name, default="⚙️") # Verbose mode: show detailed arguments, respects tool_preview_length if progress_mode == "verbose": if args: from agent.display import get_tool_preview_max_len _pl = get_tool_preview_max_len() args_str = json.dumps(args, ensure_ascii=False, default=str) # When tool_preview_length is 0 (default), don't truncate # in verbose mode — the user explicitly asked for full # detail. Platform message-length limits handle the rest. if _pl > 0 and len(args_str) > _pl: args_str = args_str[:_pl - 3] + "..." msg = f"{emoji} {tool_name}({list(args.keys())})\n{args_str}" elif preview: msg = f"{emoji} {tool_name}: \"{preview}\"" else: msg = f"{emoji} {tool_name}..." progress_queue.put(msg) return # "all" / "new" modes: short preview, respects tool_preview_length # config (defaults to 40 chars when unset to keep gateway messages # compact — unlike CLI spinners, these persist as permanent messages). if preview: from agent.display import get_tool_preview_max_len _pl = get_tool_preview_max_len() _cap = _pl if _pl > 0 else 40 if len(preview) > _cap: preview = preview[:_cap - 3] + "..." msg = f"{emoji} {tool_name}: \"{preview}\"" else: msg = f"{emoji} {tool_name}..." # Dedup: collapse consecutive identical progress messages. # Common with execute_code where models iterate with the same # code (same boilerplate imports → identical previews). if msg == last_progress_msg[0]: repeat_count[0] += 1 # Update the last line in progress_lines with a counter # via a special "dedup" queue message. progress_queue.put(("__dedup__", msg, repeat_count[0])) return last_progress_msg[0] = msg repeat_count[0] = 0 progress_queue.put(msg) # Background task to send progress messages # Accumulates tool lines into a single message that gets edited. # # Threading metadata is platform-specific: # - Slack DM threading needs event_message_id fallback (reply thread) # - Telegram uses message_thread_id only for forum topics; passing a # normal DM/group message id as thread_id causes send failures # - Other platforms should use explicit source.thread_id only if source.platform == Platform.SLACK: _progress_thread_id = source.thread_id or event_message_id else: _progress_thread_id = source.thread_id _progress_metadata = {"thread_id": _progress_thread_id} if _progress_thread_id else None async def send_progress_messages(): if not progress_queue: return adapter = self.adapters.get(source.platform) if not adapter: return # Skip tool progress for platforms that don't support message # editing (e.g. iMessage/BlueBubbles) — each progress update # would become a separate message bubble, which is noisy. if type(adapter).edit_message is BasePlatformAdapter.edit_message: while not progress_queue.empty(): try: progress_queue.get_nowait() except Exception: break return progress_lines = [] # Accumulated tool lines progress_msg_id = None # ID of the progress message to edit can_edit = True # False once an edit fails (platform doesn't support it) _last_edit_ts = 0.0 # Throttle edits to avoid Telegram flood control _PROGRESS_EDIT_INTERVAL = 1.5 # Minimum seconds between edits while True: try: if not _run_still_current(): while not progress_queue.empty(): try: progress_queue.get_nowait() except Exception: break return raw = progress_queue.get_nowait() # Drain silently when interrupted: events queued in the # window between tool parse and interrupt processing # should not render as bubbles. The "⚡ Interrupting # current task" message is sent separately and is the # last progress-flavored bubble the user should see. try: _agent_for_interrupt = agent_holder[0] if agent_holder else None if _agent_for_interrupt is not None and getattr( _agent_for_interrupt, "is_interrupted", False ): # Drop this event and continue draining. await asyncio.sleep(0) continue except Exception: pass # Handle dedup messages: update last line with repeat counter if isinstance(raw, tuple) and len(raw) == 3 and raw[0] == "__dedup__": _, base_msg, count = raw if progress_lines: progress_lines[-1] = f"{base_msg} (×{count + 1})" msg = progress_lines[-1] if progress_lines else base_msg elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": # Content bubble just landed on the platform — close off # the current tool-progress bubble so the next tool # starts a fresh bubble below the content. Without this, # tool lines keep editing the ORIGINAL progress message # above the new content, making the chat appear out of # order. Mirrors GatewayStreamConsumer.on_segment_break # on the content side. (Issue: tool + content # linearization regression after PR #7885.) progress_msg_id = None progress_lines = [] last_progress_msg[0] = None repeat_count[0] = 0 continue else: msg = raw progress_lines.append(msg) # Throttle edits: batch rapid tool updates into fewer # API calls to avoid hitting Telegram flood control. # (grammY auto-retry pattern: proactively rate-limit # instead of reacting to 429s.) _now = time.monotonic() _remaining = _PROGRESS_EDIT_INTERVAL - (_now - _last_edit_ts) if _remaining > 0: # Wait out the throttle interval, then loop back to # drain any additional queued messages before sending # a single batched edit. await asyncio.sleep(_remaining) continue if not _run_still_current(): return if can_edit and progress_msg_id is not None: # Try to edit the existing progress message full_text = "\n".join(progress_lines) result = await adapter.edit_message( chat_id=source.chat_id, message_id=progress_msg_id, content=full_text, ) if not result.success: _err = (getattr(result, "error", "") or "").lower() if "flood" in _err or "retry after" in _err: # Flood control hit — disable further edits, # switch to sending new messages only for # important updates. Don't block 23s. logger.info( "[%s] Progress edits disabled due to flood control", adapter.name, ) can_edit = False await adapter.send(chat_id=source.chat_id, content=msg, metadata=_progress_metadata) else: if can_edit: # First tool: send all accumulated text as new message full_text = "\n".join(progress_lines) result = await adapter.send(chat_id=source.chat_id, content=full_text, metadata=_progress_metadata) else: # Editing unsupported: send just this line result = await adapter.send(chat_id=source.chat_id, content=msg, metadata=_progress_metadata) if result.success and result.message_id: progress_msg_id = result.message_id _last_edit_ts = time.monotonic() # Restore typing indicator await asyncio.sleep(0.3) if _run_still_current(): await adapter.send_typing(source.chat_id, metadata=_progress_metadata) except queue.Empty: await asyncio.sleep(0.3) except asyncio.CancelledError: # Drain remaining queued messages while not progress_queue.empty(): try: raw = progress_queue.get_nowait() if isinstance(raw, tuple) and len(raw) == 3 and raw[0] == "__dedup__": _, base_msg, count = raw if progress_lines: progress_lines[-1] = f"{base_msg} (×{count + 1})" elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": # Content-bubble marker during drain: close off # the current progress bubble and start a fresh # one for any tool lines that arrived after. if can_edit and progress_lines and progress_msg_id: _pending_text = "\n".join(progress_lines) try: await adapter.edit_message( chat_id=source.chat_id, message_id=progress_msg_id, content=_pending_text, ) except Exception: pass progress_msg_id = None progress_lines = [] last_progress_msg[0] = None repeat_count[0] = 0 else: progress_lines.append(raw) except Exception: break # Final edit with all remaining tools (only if editing works) if can_edit and progress_lines and progress_msg_id: full_text = "\n".join(progress_lines) try: await adapter.edit_message( chat_id=source.chat_id, message_id=progress_msg_id, content=full_text, ) except Exception: pass return except Exception as e: logger.error("Progress message error: %s", e) await asyncio.sleep(1) # We need to share the agent instance for interrupt support agent_holder = [None] # Mutable container for the agent instance result_holder = [None] # Mutable container for the result tools_holder = [None] # Mutable container for the tool definitions stream_consumer_holder = [None] # Mutable container for stream consumer # Bridge sync step_callback → async hooks.emit for agent:step events _loop_for_step = asyncio.get_running_loop() _hooks_ref = self.hooks def _step_callback_sync(iteration: int, prev_tools: list) -> None: if not _run_still_current(): return try: # prev_tools may be list[str] or list[dict] with "name"/"result" # keys. Normalise to keep "tool_names" backward-compatible for # user-authored hooks that do ', '.join(tool_names)'. _names: list[str] = [] for _t in (prev_tools or []): if isinstance(_t, dict): _names.append(_t.get("name") or "") else: _names.append(str(_t)) asyncio.run_coroutine_threadsafe( _hooks_ref.emit("agent:step", { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "session_id": session_id, "iteration": iteration, "tool_names": _names, "tools": prev_tools, }), _loop_for_step, ) except Exception as _e: logger.debug("agent:step hook error: %s", _e) # Bridge sync status_callback → async adapter.send for context pressure _status_adapter = self.adapters.get(source.platform) _status_chat_id = source.chat_id _status_thread_metadata = {"thread_id": _progress_thread_id} if _progress_thread_id else None def _status_callback_sync(event_type: str, message: str) -> None: if not _status_adapter or not _run_still_current(): return try: asyncio.run_coroutine_threadsafe( _status_adapter.send( _status_chat_id, message, metadata=_status_thread_metadata, ), _loop_for_step, ) except Exception as _e: logger.debug("status_callback error (%s): %s", event_type, _e) def run_sync(): # The conditional re-assignment of `message` further below # (prepending model-switch notes) makes Python treat it as a # local variable in the entire function. `nonlocal` lets us # read *and* reassign the outer `_run_agent` parameter without # triggering an UnboundLocalError on the earlier read at # `_resolve_turn_agent_config(message, …)`. nonlocal message # session_key is now set via contextvars in _set_session_env() # (concurrency-safe). Keep os.environ as fallback for CLI/cron. os.environ["HERMES_SESSION_KEY"] = session_key or "" # Read from env var or use default (same as CLI) max_iterations = int(os.getenv("HERMES_MAX_ITERATIONS", "90")) # Map platform enum to the platform hint key the agent understands. # Platform.LOCAL ("local") maps to "cli"; others pass through as-is. platform_key = "cli" if source.platform == Platform.LOCAL else source.platform.value # Combine platform context, per-channel context, and the user-configured # ephemeral system prompt. combined_ephemeral = context_prompt or "" event_channel_prompt = (channel_prompt or "").strip() if event_channel_prompt: combined_ephemeral = (combined_ephemeral + "\n\n" + event_channel_prompt).strip() if self._ephemeral_system_prompt: combined_ephemeral = (combined_ephemeral + "\n\n" + self._ephemeral_system_prompt).strip() # Re-read .env and config for fresh credentials (gateway is long-lived, # keys may change without restart). try: load_dotenv(_env_path, override=True, encoding="utf-8") except UnicodeDecodeError: load_dotenv(_env_path, override=True, encoding="latin-1") except Exception: pass try: model, runtime_kwargs = self._resolve_session_agent_runtime( source=source, session_key=session_key, user_config=user_config, ) logger.debug( "run_agent resolved: model=%s provider=%s session=%s", model, runtime_kwargs.get("provider"), session_key or "", ) except Exception as exc: return { "final_response": f"⚠️ Provider authentication failed: {exc}", "messages": [], "api_calls": 0, "tools": [], } pr = self._provider_routing reasoning_config = self._resolve_session_reasoning_config( source=source, session_key=session_key, ) self._reasoning_config = reasoning_config self._service_tier = self._load_service_tier() # Set up stream consumer for token streaming or interim commentary. _stream_consumer = None _stream_delta_cb = None _scfg = getattr(getattr(self, 'config', None), 'streaming', None) if _scfg is None: from gateway.config import StreamingConfig _scfg = StreamingConfig() # Per-platform streaming gate: display.platforms..streaming # can disable streaming for specific platforms even when the global # streaming config is enabled. _plat_streaming = resolve_display_setting( user_config, platform_key, "streaming" ) # None = no per-platform override → follow global config _streaming_enabled = ( _scfg.enabled and _scfg.transport != "off" if _plat_streaming is None else bool(_plat_streaming) ) _want_stream_deltas = _streaming_enabled _want_interim_messages = interim_assistant_messages_enabled _want_interim_consumer = _want_interim_messages if _want_stream_deltas or _want_interim_consumer: try: from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig _adapter = self.adapters.get(source.platform) if _adapter: # Platforms that don't support editing sent messages # (e.g. QQ, WeChat) should skip streaming entirely — # without edit support, the consumer sends a partial # first message that can never be updated, resulting in # duplicate messages (partial + final). _adapter_supports_edit = getattr(_adapter, "SUPPORTS_MESSAGE_EDITING", True) if not _adapter_supports_edit: raise RuntimeError("skip streaming for non-editable platform") _effective_cursor = _scfg.cursor # Some Matrix clients render the streaming cursor # as a visible tofu/white-box artifact. Keep # streaming text on Matrix, but suppress the cursor. _buffer_only = False if source.platform == Platform.MATRIX: _effective_cursor = "" _buffer_only = True # Fresh-final applies to Telegram only — other # platforms either edit in place cheaply or don't # have the edit-timestamp-stays-stale problem. # (Ported from openclaw/openclaw#72038.) _fresh_final_secs = ( float(getattr(_scfg, "fresh_final_after_seconds", 0.0) or 0.0) if source.platform == Platform.TELEGRAM else 0.0 ) _consumer_cfg = StreamConsumerConfig( edit_interval=_scfg.edit_interval, buffer_threshold=_scfg.buffer_threshold, cursor=_effective_cursor, buffer_only=_buffer_only, fresh_final_after_seconds=_fresh_final_secs, ) _stream_consumer = GatewayStreamConsumer( adapter=_adapter, chat_id=source.chat_id, config=_consumer_cfg, metadata={"thread_id": _progress_thread_id} if _progress_thread_id else None, on_new_message=( (lambda: progress_queue.put(("__reset__",))) if progress_queue is not None else None ), ) if _want_stream_deltas: def _stream_delta_cb(text: str) -> None: if _run_still_current(): _stream_consumer.on_delta(text) stream_consumer_holder[0] = _stream_consumer except Exception as _sc_err: logger.debug("Could not set up stream consumer: %s", _sc_err) def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: if not _run_still_current(): return if _stream_consumer is not None: if already_streamed: _stream_consumer.on_segment_break() else: _stream_consumer.on_commentary(text) return if already_streamed or not _status_adapter or not str(text or "").strip(): return try: asyncio.run_coroutine_threadsafe( _status_adapter.send( _status_chat_id, text, metadata=_status_thread_metadata, ), _loop_for_step, ) except Exception as _e: logger.debug("interim_assistant_callback error: %s", _e) turn_route = self._resolve_turn_agent_config(message, model, runtime_kwargs) # Check agent cache — reuse the AIAgent from the previous message # in this session to preserve the frozen system prompt and tool # schemas for prompt cache hits. _sig = self._agent_config_signature( turn_route["model"], turn_route["runtime"], enabled_toolsets, combined_ephemeral, cache_keys=self._extract_cache_busting_config(user_config), ) agent = None _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock and _cache is not None: with _cache_lock: cached = _cache.get(session_key) if cached and cached[1] == _sig: agent = cached[0] # Refresh LRU order so the cap enforcement evicts # truly-oldest entries, not the one we just used. if hasattr(_cache, "move_to_end"): try: _cache.move_to_end(session_key) except KeyError: pass self._init_cached_agent_for_turn(agent, _interrupt_depth) logger.debug("Reusing cached agent for session %s", session_key) if agent is None: # Config changed or first message — create fresh agent agent = AIAgent( model=turn_route["model"], **turn_route["runtime"], max_iterations=max_iterations, quiet_mode=True, verbose_logging=False, enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, ephemeral_system_prompt=combined_ephemeral or None, prefill_messages=self._prefill_messages or None, reasoning_config=reasoning_config, service_tier=self._service_tier, request_overrides=turn_route.get("request_overrides"), providers_allowed=pr.get("only"), providers_ignored=pr.get("ignore"), providers_order=pr.get("order"), provider_sort=pr.get("sort"), provider_require_parameters=pr.get("require_parameters", False), provider_data_collection=pr.get("data_collection"), session_id=session_id, platform=platform_key, user_id=source.user_id, user_name=source.user_name, chat_id=source.chat_id, chat_name=source.chat_name, chat_type=source.chat_type, thread_id=source.thread_id, gateway_session_key=session_key, session_db=self._session_db, fallback_model=self._fallback_model, ) if _cache_lock and _cache is not None: with _cache_lock: _cache[session_key] = (agent, _sig) self._enforce_agent_cache_cap() logger.debug("Created new agent for session %s (sig=%s)", session_key, _sig) # Per-message state — callbacks and reasoning config change every # turn and must not be baked into the cached agent constructor. agent.tool_progress_callback = progress_callback if tool_progress_enabled else None agent.step_callback = _step_callback_sync if _hooks_ref.loaded_hooks else None agent.stream_delta_callback = _stream_delta_cb agent.interim_assistant_callback = _interim_assistant_cb if _want_interim_messages else None agent.status_callback = _status_callback_sync agent.reasoning_config = reasoning_config agent.service_tier = self._service_tier agent.request_overrides = turn_route.get("request_overrides") or {} _bg_review_release = threading.Event() _bg_review_pending: list[str] = [] _bg_review_pending_lock = threading.Lock() def _deliver_bg_review_message(message: str) -> None: if not _status_adapter or not _run_still_current(): return try: asyncio.run_coroutine_threadsafe( _status_adapter.send( _status_chat_id, message, metadata=_status_thread_metadata, ), _loop_for_step, ) except Exception as _e: logger.debug("background_review_callback error: %s", _e) def _release_bg_review_messages() -> None: _bg_review_release.set() with _bg_review_pending_lock: pending = list(_bg_review_pending) _bg_review_pending.clear() for queued in pending: _deliver_bg_review_message(queued) # Background review delivery — send "💾 Memory updated" etc. to user def _bg_review_send(message: str) -> None: if not _status_adapter or not _run_still_current(): return if not _bg_review_release.is_set(): with _bg_review_pending_lock: if not _bg_review_release.is_set(): _bg_review_pending.append(message) return _deliver_bg_review_message(message) agent.background_review_callback = _bg_review_send # Register the release hook on the adapter so base.py's finally # block can fire it after delivering the main response. if _status_adapter and session_key: if getattr(type(_status_adapter), "register_post_delivery_callback", None) is not None: _status_adapter.register_post_delivery_callback( session_key, _release_bg_review_messages, generation=run_generation, ) else: _pdc = getattr(_status_adapter, "_post_delivery_callbacks", None) if _pdc is not None: _pdc[session_key] = _release_bg_review_messages # Store agent reference for interrupt support agent_holder[0] = agent # Capture the full tool definitions for transcript logging tools_holder[0] = agent.tools if hasattr(agent, 'tools') else None # Convert history to agent format. # Two cases: # 1. Normal path (from transcript): simple {role, content, timestamp} dicts # - Strip timestamps, keep role+content # 2. Interrupt path (from agent result["messages"]): full agent messages # that may include tool_calls, tool_call_id, reasoning, etc. # - These must be passed through intact so the API sees valid # assistant→tool sequences (dropping tool_calls causes 500 errors) agent_history = [] for msg in history: role = msg.get("role") if not role: continue # Skip metadata entries (tool definitions, session info) # -- these are for transcript logging, not for the LLM if role in ("session_meta",): continue # Skip system messages -- the agent rebuilds its own system prompt if role == "system": continue # Rich agent messages (tool_calls, tool results) must be passed # through intact so the API sees valid assistant→tool sequences has_tool_calls = "tool_calls" in msg has_tool_call_id = "tool_call_id" in msg is_tool_message = role == "tool" if has_tool_calls or has_tool_call_id or is_tool_message: clean_msg = {k: v for k, v in msg.items() if k != "timestamp"} agent_history.append(clean_msg) else: # Simple text message - just need role and content content = msg.get("content") if content: # Tag cross-platform mirror messages so the agent knows their origin if msg.get("mirror"): mirror_src = msg.get("mirror_source", "another session") content = f"[Delivered from {mirror_src}] {content}" entry = {"role": role, "content": content} # Preserve reasoning fields on assistant messages so # multi-turn reasoning context survives session reload. # The agent's _build_api_kwargs converts these to the # provider-specific format (reasoning_content, etc.). if role == "assistant": for _rkey in ("reasoning", "reasoning_details", "codex_reasoning_items"): _rval = msg.get(_rkey) if _rval: entry[_rkey] = _rval agent_history.append(entry) # Collect MEDIA paths already in history so we can exclude them # from the current turn's extraction. This is compression-safe: # even if the message list shrinks, we know which paths are old. _history_media_paths: set = set() for _hm in agent_history: if _hm.get("role") in ("tool", "function"): _hc = _hm.get("content", "") if "MEDIA:" in _hc: for _match in re.finditer(r'MEDIA:(\S+)', _hc): _p = _match.group(1).strip().rstrip('",}') if _p: _history_media_paths.add(_p) # Register per-session gateway approval callback so dangerous # command approval blocks the agent thread (mirrors CLI input()). # The callback bridges sync→async to send the approval request # to the user immediately. from tools.approval import ( register_gateway_notify, reset_current_session_key, set_current_session_key, unregister_gateway_notify, ) def _approval_notify_sync(approval_data: dict) -> None: """Send the approval request to the user from the agent thread. If the adapter supports interactive button-based approvals (e.g. Discord's ``send_exec_approval``), use that for a richer UX. Otherwise fall back to a plain text message with ``/approve`` instructions. """ # Pause the typing indicator while the agent waits for # user approval. Critical for Slack's Assistant API where # assistant_threads_setStatus disables the compose box — the # user literally cannot type /approve while "is thinking..." # is active. The approval message send auto-clears the Slack # status; pausing prevents _keep_typing from re-setting it. # Typing resumes in _handle_approve_command/_handle_deny_command. _status_adapter.pause_typing_for_chat(_status_chat_id) cmd = approval_data.get("command", "") desc = approval_data.get("description", "dangerous command") # Prefer button-based approval when the adapter supports it. # Check the *class* for the method, not the instance — avoids # false positives from MagicMock auto-attribute creation in tests. if getattr(type(_status_adapter), "send_exec_approval", None) is not None: try: _approval_result = asyncio.run_coroutine_threadsafe( _status_adapter.send_exec_approval( chat_id=_status_chat_id, command=cmd, session_key=_approval_session_key, description=desc, metadata=_status_thread_metadata, ), _loop_for_step, ).result(timeout=15) if _approval_result.success: return logger.warning( "Button-based approval failed (send returned error), falling back to text: %s", _approval_result.error, ) except Exception as _e: logger.warning( "Button-based approval failed, falling back to text: %s", _e ) # Fallback: plain text approval prompt cmd_preview = cmd[:200] + "..." if len(cmd) > 200 else cmd msg = ( f"⚠️ **Dangerous command requires approval:**\n" f"```\n{cmd_preview}\n```\n" f"Reason: {desc}\n\n" f"Reply `/approve` to execute, `/approve session` to approve this pattern " f"for the session, `/approve always` to approve permanently, or `/deny` to cancel." ) try: asyncio.run_coroutine_threadsafe( _status_adapter.send( _status_chat_id, msg, metadata=_status_thread_metadata, ), _loop_for_step, ).result(timeout=15) except Exception as _e: logger.error("Failed to send approval request: %s", _e) # Prepend pending model switch note so the model knows about the switch _pending_notes = getattr(self, '_pending_model_notes', {}) _msn = _pending_notes.pop(session_key, None) if session_key else None if _msn: message = _msn + "\n\n" + message # Auto-continue: if the loaded history ends with a tool result, # the previous agent turn was interrupted mid-work (gateway # restart, crash, SIGTERM). Prepend a system note so the model # finishes processing the pending tool results before addressing # the user's new message. (#4493) # # Session-level resume_pending (set on drain-timeout shutdown) # escalates the wording — the transcript's last role may be # anything (tool, assistant with unfinished work, etc.), so we # give a stronger, reason-aware instruction that subsumes the # tool-tail case. # # Freshness gate (#16802): both branches are gated on the age # of the last persisted transcript row. That is the correct # "when did we last do anything here" signal for both the # resume_pending path (restart watchdog) and the tool-tail # path (in-flight tool loop killed). We read ``history[-1]`` # here because ``agent_history`` has already stripped the # ``timestamp`` field off tool/tool_call rows for API purity # (see the `k != "timestamp"` filter above). Rows without a # timestamp (legacy transcripts) are treated as fresh so the # historical auto-continue behaviour is preserved. _freshness_window = _auto_continue_freshness_window() _interruption_is_fresh = _is_fresh_gateway_interruption( _last_transcript_timestamp(history), window_secs=_freshness_window, ) _resume_entry = None if session_key: try: _resume_entry = self.session_store._entries.get(session_key) except Exception: _resume_entry = None _is_resume_pending = bool( _resume_entry is not None and getattr(_resume_entry, "resume_pending", False) and _interruption_is_fresh ) _has_fresh_tool_tail = bool( agent_history and agent_history[-1].get("role") == "tool" and _interruption_is_fresh ) if _is_resume_pending: _reason = getattr(_resume_entry, "resume_reason", None) or "restart_timeout" _reason_phrase = ( "a gateway restart" if _reason == "restart_timeout" else "a gateway shutdown" if _reason == "shutdown_timeout" else "a gateway interruption" ) message = ( f"[System note: Your previous turn in this session was interrupted " f"by {_reason_phrase}. The conversation history below is intact. " f"If it contains unfinished tool result(s), process them first and " f"summarize what was accomplished, then address the user's new " f"message below.]\n\n" + message ) elif _has_fresh_tool_tail: message = ( "[System note: Your previous turn was interrupted before you could " "process the last tool result(s). The conversation history contains " "tool outputs you haven't responded to yet. Please finish processing " "those results and summarize what was accomplished, then address the " "user's new message below.]\n\n" + message ) # Consume one-shot /reload-skills note (if the user ran # /reload-skills since their last turn in this session). Same # queue pattern as CLI: prepend to the NEXT user message, then # clear. Nothing was written to the transcript out-of-band, so # message alternation stays intact. _pending_notes = getattr(self, "_pending_skills_reload_notes", None) if _pending_notes and session_key and session_key in _pending_notes: _srn = _pending_notes.pop(session_key, None) if _srn: message = _srn + "\n\n" + message _approval_session_key = session_key or "" _approval_session_token = set_current_session_key(_approval_session_key) register_gateway_notify(_approval_session_key, _approval_notify_sync) try: # If _prepare_inbound_message_text buffered image paths for native # attachment, wrap the user turn as an OpenAI-style multimodal # content list. Consume-and-clear so subsequent turns on the same # runner instance don't re-attach stale images. _native_imgs = self._consume_pending_native_image_paths(session_key) if _native_imgs: try: from agent.image_routing import build_native_content_parts _parts, _skipped = build_native_content_parts( message, _native_imgs, ) if _skipped: logger.warning( "Native image attachment: skipped %d unreadable path(s): %s", len(_skipped), _skipped, ) if any(p.get("type") == "image_url" for p in _parts): _run_message: Any = _parts else: # All images failed to read — fall back to plain text. _run_message = message except Exception as _img_exc: logger.warning( "Native image attachment failed, falling back to text: %s", _img_exc, ) _run_message = message else: _run_message = message result = agent.run_conversation(_run_message, conversation_history=agent_history, task_id=session_id) finally: unregister_gateway_notify(_approval_session_key) reset_current_session_key(_approval_session_token) result_holder[0] = result # Signal the stream consumer that the agent is done if _stream_consumer is not None: _stream_consumer.finish() # Return final response, or a message if something went wrong final_response = result.get("final_response") # Extract actual token counts from the agent instance used for this run _last_prompt_toks = 0 _input_toks = 0 _output_toks = 0 _context_length = 0 _agent = agent_holder[0] if _agent and hasattr(_agent, "context_compressor"): _last_prompt_toks = getattr(_agent.context_compressor, "last_prompt_tokens", 0) _input_toks = getattr(_agent, "session_prompt_tokens", 0) _output_toks = getattr(_agent, "session_completion_tokens", 0) _context_length = getattr(_agent.context_compressor, "context_length", 0) or 0 _resolved_model = getattr(_agent, "model", None) if _agent else None if not final_response: error_msg = f"⚠️ {result['error']}" if result.get("error") else "" return { "final_response": error_msg, "messages": result.get("messages", []), "api_calls": result.get("api_calls", 0), "failed": result.get("failed", False), "compression_exhausted": result.get("compression_exhausted", False), "tools": tools_holder[0] or [], "history_offset": len(agent_history), "last_prompt_tokens": _last_prompt_toks, "input_tokens": _input_toks, "output_tokens": _output_toks, "model": _resolved_model, "context_length": _context_length, } # Scan tool results for MEDIA: tags that need to be delivered # as native audio/file attachments. The TTS tool embeds MEDIA: tags # in its JSON response, but the model's final text reply usually # doesn't include them. We collect unique tags from tool results and # append any that aren't already present in the final response, so the # adapter's extract_media() can find and deliver the files exactly once. # # Uses path-based deduplication against _history_media_paths (collected # before run_conversation) instead of index slicing. This is safe even # when context compression shrinks the message list. (Fixes #160) if "MEDIA:" not in final_response: media_tags = [] has_voice_directive = False for msg in result.get("messages", []): if msg.get("role") in ("tool", "function"): content = msg.get("content", "") if "MEDIA:" in content: for match in re.finditer(r'MEDIA:(\S+)', content): path = match.group(1).strip().rstrip('",}') if path and path not in _history_media_paths: media_tags.append(f"MEDIA:{path}") if "[[audio_as_voice]]" in content: has_voice_directive = True if media_tags: seen = set() unique_tags = [] for tag in media_tags: if tag not in seen: seen.add(tag) unique_tags.append(tag) if has_voice_directive: unique_tags.insert(0, "[[audio_as_voice]]") final_response = final_response + "\n" + "\n".join(unique_tags) # Sync session_id: the agent may have created a new session during # mid-run context compression (_compress_context splits sessions). # If so, update the session store entry so the NEXT message loads # the compressed transcript, not the stale pre-compression one. agent = agent_holder[0] _session_was_split = False if agent and session_key and hasattr(agent, 'session_id') and agent.session_id != session_id: _session_was_split = True logger.info( "Session split detected: %s → %s (compression)", session_id, agent.session_id, ) entry = self.session_store._entries.get(session_key) if entry: entry.session_id = agent.session_id self.session_store._save() effective_session_id = getattr(agent, 'session_id', session_id) if agent else session_id # When compression created a new session, the messages list was # shortened. Using the original history offset would produce an # empty new_messages slice, causing the gateway to write only a # user/assistant pair — losing the compressed summary and tail. # Reset to 0 so the gateway writes ALL compressed messages. _effective_history_offset = 0 if _session_was_split else len(agent_history) # Auto-generate session title after first exchange (non-blocking) if final_response and self._session_db: try: from agent.title_generator import maybe_auto_title all_msgs = result_holder[0].get("messages", []) if result_holder[0] else [] # Route title-generation failures through the agent's # user-visible warning channel so a depleted auxiliary # provider doesn't silently leave sessions untitled # (issue #15775). _title_failure_cb = getattr( agent, "_emit_auxiliary_failure", None ) maybe_auto_title( self._session_db, effective_session_id, message, final_response, all_msgs, failure_callback=_title_failure_cb, main_runtime={ "model": getattr(agent, "model", None), "provider": getattr(agent, "provider", None), "base_url": getattr(agent, "base_url", None), "api_key": getattr(agent, "api_key", None), "api_mode": getattr(agent, "api_mode", None), } if agent else None, ) except Exception: pass return { "final_response": final_response, "last_reasoning": result.get("last_reasoning"), "messages": result_holder[0].get("messages", []) if result_holder[0] else [], "api_calls": result_holder[0].get("api_calls", 0) if result_holder[0] else 0, "tools": tools_holder[0] or [], "history_offset": _effective_history_offset, "last_prompt_tokens": _last_prompt_toks, "input_tokens": _input_toks, "output_tokens": _output_toks, "model": _resolved_model, "context_length": _context_length, "session_id": effective_session_id, "response_previewed": result.get("response_previewed", False), } # Start progress message sender if enabled progress_task = None if tool_progress_enabled: progress_task = asyncio.create_task(send_progress_messages()) # Start stream consumer task — polls for consumer creation since it # happens inside run_sync (thread pool) after the agent is constructed. stream_task = None async def _start_stream_consumer(): """Wait for the stream consumer to be created, then run it.""" for _ in range(200): # Up to 10s wait if stream_consumer_holder[0] is not None: await stream_consumer_holder[0].run() return await asyncio.sleep(0.05) stream_task = asyncio.create_task(_start_stream_consumer()) # Track this agent as running for this session (for interrupt support) # We do this in a callback after the agent is created async def track_agent(): # Wait for agent to be created while agent_holder[0] is None: await asyncio.sleep(0.05) if not session_key: return # Only promote the sentinel to the real agent if this run is still # current. If /stop or /new bumped the generation while we were # spinning up, leave the newer run's slot alone — we'll be # discarded by the stale-result check in _handle_message_with_agent. if run_generation is not None and not self._is_session_run_current( session_key, run_generation ): logger.info( "Skipping stale agent promotion for %s — generation %s is no longer current", session_key or "", run_generation, ) return self._running_agents[session_key] = agent_holder[0] if self._draining: self._update_runtime_status("draining") tracking_task = asyncio.create_task(track_agent()) # Monitor for interrupts from the adapter (new messages arriving). # This is the PRIMARY interrupt path for regular text messages — # Level 1 (base.py) catches them before _handle_message() is reached, # so the Level 2 running_agent.interrupt() path never fires. # The inactivity poll loop below has a BACKUP check in case this # task dies (no error handling = silent death = lost interrupts). _interrupt_detected = asyncio.Event() # shared with backup check async def monitor_for_interrupt(): if not session_key: return while True: await asyncio.sleep(0.2) # Check every 200ms try: # Re-resolve adapter each iteration so reconnects don't # leave us holding a stale reference. _adapter = self.adapters.get(source.platform) if not _adapter: continue # Check if adapter has a pending interrupt for this session. # Must use session_key (build_session_key output) — NOT # source.chat_id — because the adapter stores interrupt events # under the full session key. if hasattr(_adapter, 'has_pending_interrupt') and _adapter.has_pending_interrupt(session_key): agent = agent_holder[0] if agent: # Peek at the pending message text WITHOUT consuming it. # The message must remain in _pending_messages so the # post-run dequeue at _dequeue_pending_event() can # retrieve the full MessageEvent (with media metadata). # If we pop here, a race exists: the agent may finish # before checking _interrupt_requested, and the message # is lost — neither the interrupt path nor the dequeue # path finds it. _peek_event = _adapter._pending_messages.get(session_key) pending_text = _peek_event.text if _peek_event else None logger.debug("Interrupt detected from adapter, signaling agent...") agent.interrupt(pending_text) _interrupt_detected.set() break except asyncio.CancelledError: raise except Exception as _mon_err: logger.debug("monitor_for_interrupt error (will retry): %s", _mon_err) interrupt_monitor = asyncio.create_task(monitor_for_interrupt()) # Periodic "still working" notifications for long-running tasks. # Fires every N seconds so the user knows the agent hasn't died. # Config: agent.gateway_notify_interval in config.yaml, or # HERMES_AGENT_NOTIFY_INTERVAL env var. Default 180s (3 min). # 0 = disable notifications. _NOTIFY_INTERVAL_RAW = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) _NOTIFY_INTERVAL = _NOTIFY_INTERVAL_RAW if _NOTIFY_INTERVAL_RAW > 0 else None _notify_start = time.time() async def _notify_long_running(): if _NOTIFY_INTERVAL is None: return # Notifications disabled (gateway_notify_interval: 0) _notify_adapter = self.adapters.get(source.platform) if not _notify_adapter: return while True: await asyncio.sleep(_NOTIFY_INTERVAL) _elapsed_mins = int((time.time() - _notify_start) // 60) # Include agent activity context if available. _agent_ref = agent_holder[0] _status_detail = "" if _agent_ref and hasattr(_agent_ref, "get_activity_summary"): try: _a = _agent_ref.get_activity_summary() _parts = [f"iteration {_a['api_call_count']}/{_a['max_iterations']}"] if _a.get("current_tool"): _parts.append(f"running: {_a['current_tool']}") else: _parts.append(_a.get("last_activity_desc", "")) _status_detail = " — " + ", ".join(_parts) except Exception: pass try: await _notify_adapter.send( source.chat_id, f"⏳ Still working... ({_elapsed_mins} min elapsed{_status_detail})", metadata=_status_thread_metadata, ) except Exception as _ne: logger.debug("Long-running notification error: %s", _ne) _notify_task = asyncio.create_task(_notify_long_running()) try: # Run in thread pool to not block. Use an *inactivity*-based # timeout instead of a wall-clock limit: the agent can run for # hours if it's actively calling tools / receiving stream tokens, # but a hung API call or stuck tool with no activity for the # configured duration is caught and killed. (#4815) # # Config: agent.gateway_timeout in config.yaml, or # HERMES_AGENT_TIMEOUT env var (env var takes precedence). # Default 1800s (30 min inactivity). 0 = unlimited. _agent_timeout_raw = _float_env("HERMES_AGENT_TIMEOUT", 1800) _agent_timeout = _agent_timeout_raw if _agent_timeout_raw > 0 else None _agent_warning_raw = _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900) _agent_warning = _agent_warning_raw if _agent_warning_raw > 0 else None _warning_fired = False _executor_task = asyncio.ensure_future( self._run_in_executor_with_context(run_sync) ) _inactivity_timeout = False _POLL_INTERVAL = 5.0 if _agent_timeout is None: # Unlimited — still poll periodically for backup interrupt # detection in case monitor_for_interrupt() silently died. response = None while True: done, _ = await asyncio.wait( {_executor_task}, timeout=_POLL_INTERVAL ) if done: response = _executor_task.result() break # Backup interrupt check: if the monitor task died or # missed the interrupt, catch it here. if not _interrupt_detected.is_set() and session_key: _backup_adapter = self.adapters.get(source.platform) _backup_agent = agent_holder[0] if (_backup_adapter and _backup_agent and hasattr(_backup_adapter, 'has_pending_interrupt') and _backup_adapter.has_pending_interrupt(session_key)): _bp_event = _backup_adapter._pending_messages.get(session_key) _bp_text = _bp_event.text if _bp_event else None logger.info( "Backup interrupt detected for session %s " "(monitor task state: %s)", session_key, "done" if interrupt_monitor.done() else "running", ) _backup_agent.interrupt(_bp_text) _interrupt_detected.set() else: # Poll loop: check the agent's built-in activity tracker # (updated by _touch_activity() on every tool call, API # call, and stream delta) every few seconds. response = None while True: done, _ = await asyncio.wait( {_executor_task}, timeout=_POLL_INTERVAL ) if done: response = _executor_task.result() break # Agent still running — check inactivity. _agent_ref = agent_holder[0] _idle_secs = 0.0 if _agent_ref and hasattr(_agent_ref, "get_activity_summary"): try: _act = _agent_ref.get_activity_summary() _idle_secs = _act.get("seconds_since_activity", 0.0) except Exception: pass # Staged warning: fire once before escalating to full timeout. if (not _warning_fired and _agent_warning is not None and _idle_secs >= _agent_warning): _warning_fired = True _warn_adapter = self.adapters.get(source.platform) if _warn_adapter: _elapsed_warn = int(_agent_warning // 60) or 1 _remaining_mins = int((_agent_timeout - _agent_warning) // 60) or 1 try: await _warn_adapter.send( source.chat_id, f"⚠️ No activity for {_elapsed_warn} min. " f"If the agent does not respond soon, it will " f"be timed out in {_remaining_mins} min. " f"You can continue waiting or use /reset.", metadata=_status_thread_metadata, ) except Exception as _warn_err: logger.debug("Inactivity warning send error: %s", _warn_err) if _idle_secs >= _agent_timeout: _inactivity_timeout = True break # Backup interrupt check (same as unlimited path). if not _interrupt_detected.is_set() and session_key: _backup_adapter = self.adapters.get(source.platform) _backup_agent = agent_holder[0] if (_backup_adapter and _backup_agent and hasattr(_backup_adapter, 'has_pending_interrupt') and _backup_adapter.has_pending_interrupt(session_key)): _bp_event = _backup_adapter._pending_messages.get(session_key) _bp_text = _bp_event.text if _bp_event else None logger.info( "Backup interrupt detected for session %s " "(monitor task state: %s)", session_key, "done" if interrupt_monitor.done() else "running", ) _backup_agent.interrupt(_bp_text) _interrupt_detected.set() if _inactivity_timeout: # Build a diagnostic summary from the agent's activity tracker. _timed_out_agent = agent_holder[0] _activity = {} if _timed_out_agent and hasattr(_timed_out_agent, "get_activity_summary"): try: _activity = _timed_out_agent.get_activity_summary() except Exception: pass _last_desc = _activity.get("last_activity_desc", "unknown") _secs_ago = _activity.get("seconds_since_activity", 0) _cur_tool = _activity.get("current_tool") _iter_n = _activity.get("api_call_count", 0) _iter_max = _activity.get("max_iterations", 0) logger.error( "Agent idle for %.0fs (timeout %.0fs) in session %s " "| last_activity=%s | iteration=%s/%s | tool=%s", _secs_ago, _agent_timeout, session_key, _last_desc, _iter_n, _iter_max, _cur_tool or "none", ) # Interrupt the agent if it's still running so the thread # pool worker is freed. if _timed_out_agent and hasattr(_timed_out_agent, "interrupt"): _timed_out_agent.interrupt(_INTERRUPT_REASON_TIMEOUT) _timeout_mins = int(_agent_timeout // 60) or 1 # Construct a user-facing message with diagnostic context. _diag_lines = [ f"⏱️ Agent inactive for {_timeout_mins} min — no tool calls " f"or API responses." ] if _cur_tool: _diag_lines.append( f"The agent appears stuck on tool `{_cur_tool}` " f"({_secs_ago:.0f}s since last activity, " f"iteration {_iter_n}/{_iter_max})." ) else: _diag_lines.append( f"Last activity: {_last_desc} ({_secs_ago:.0f}s ago, " f"iteration {_iter_n}/{_iter_max}). " "The agent may have been waiting on an API response." ) _diag_lines.append( "To increase the limit, set agent.gateway_timeout in config.yaml " "(value in seconds, 0 = no limit) and restart the gateway.\n" "Try again, or use /reset to start fresh." ) response = { "final_response": "\n".join(_diag_lines), "messages": result_holder[0].get("messages", []) if result_holder[0] else [], "api_calls": _iter_n, "tools": tools_holder[0] or [], "history_offset": 0, "failed": True, } # Track fallback model state: if the agent switched to a # fallback model during this run, persist it so /model shows # the actually-active model instead of the config default. # Skip eviction when the run failed — evicting a failed agent # forces MCP reinit on the next message for no benefit (the # same error will recur). This was the root cause of #7130: # a bad model ID triggered fallback → eviction → recreation → # MCP reinit → same 400 → loop, burning 91% CPU for hours. _agent = agent_holder[0] _result_for_fb = result_holder[0] _run_failed = _result_for_fb.get("failed") if _result_for_fb else False if _agent is not None and hasattr(_agent, 'model') and not _run_failed: _cfg_model = _resolve_gateway_model() if _agent.model != _cfg_model and not self._is_intentional_model_switch(session_key, _agent.model): # Fallback activated on a successful run — evict cached # agent so the next message retries the primary model. self._evict_cached_agent(session_key) # Check if we were interrupted OR have a queued message (/queue). result = result_holder[0] adapter = self.adapters.get(source.platform) # Get pending message from adapter. # Use session_key (not source.chat_id) to match adapter's storage keys. pending_event = None pending = None if result and adapter and session_key: pending_event = _dequeue_pending_event(adapter, session_key) # /queue overflow: after consuming the adapter's "next-up" # slot, promote the next queued event into it so the # recursive run's drain will see it. This keeps the slot # occupied for the full FIFO chain, which (a) preserves # order, and (b) causes any mid-chain /queue to correctly # route to overflow rather than jumping the queue. pending_event = self._promote_queued_event(session_key, adapter, pending_event) if result.get("interrupted") and not pending_event and result.get("interrupt_message"): interrupt_message = result.get("interrupt_message") if _is_control_interrupt_message(interrupt_message): logger.info( "Ignoring control interrupt message for session %s: %s", session_key or "?", interrupt_message, ) else: pending = interrupt_message elif pending_event: pending = pending_event.text or _build_media_placeholder(pending_event) logger.debug("Processing queued message after agent completion: '%s...'", pending[:40]) # Leftover /steer: if a steer arrived after the last tool batch # (e.g. during the final API call), the agent couldn't inject it # and returned it in result["pending_steer"]. Deliver it as the # next user turn so it isn't silently dropped. if result and not pending and not pending_event: _leftover_steer = result.get("pending_steer") if _leftover_steer: pending = _leftover_steer logger.debug("Delivering leftover /steer as next turn: '%s...'", pending[:40]) # Safety net: if the pending text is a slash command (e.g. "/stop", # "/new"), discard it — commands should never be passed to the agent # as user input. The primary fix is in base.py (commands bypass the # active-session guard), but this catches edge cases where command # text leaks through the interrupt_message fallback. if pending and pending.strip().startswith("/"): _pending_parts = pending.strip().split(None, 1) _pending_cmd_word = _pending_parts[0][1:].lower() if _pending_parts else "" if _pending_cmd_word: try: from hermes_cli.commands import resolve_command as _rc_pending if _rc_pending(_pending_cmd_word): logger.info( "Discarding command '/%s' from pending queue — " "commands must not be passed as agent input", _pending_cmd_word, ) pending_event = None pending = None except Exception: pass if self._draining and (pending_event or pending): logger.info( "Discarding pending follow-up for session %s during gateway %s", session_key or "?", self._status_action_label(), ) pending_event = None pending = None if pending_event or pending: logger.debug("Processing pending message: '%s...'", pending[:40]) # Clear the adapter's interrupt event so the next _run_agent call # doesn't immediately re-trigger the interrupt before the new agent # even makes its first API call (this was causing an infinite loop). if adapter and hasattr(adapter, '_active_sessions') and session_key and session_key in adapter._active_sessions: adapter._active_sessions[session_key].clear() # Cap recursion depth to prevent resource exhaustion when the # user sends multiple messages while the agent keeps failing. (#816) if _interrupt_depth >= self._MAX_INTERRUPT_DEPTH: logger.warning( "Interrupt recursion depth %d reached for session %s — " "queueing message instead of recursing.", _interrupt_depth, session_key, ) adapter = self.adapters.get(source.platform) if adapter and pending_event: merge_pending_message_event(adapter._pending_messages, session_key, pending_event) elif adapter and hasattr(adapter, 'queue_message'): adapter.queue_message(session_key, pending) return result_holder[0] or {"final_response": response, "messages": history} was_interrupted = result.get("interrupted") if not was_interrupted: # Queued message after normal completion — deliver the first # response before processing the queued follow-up. # Skip if streaming already delivered it. _sc = stream_consumer_holder[0] if _sc and stream_task: try: await asyncio.wait_for(stream_task, timeout=5.0) except (asyncio.TimeoutError, asyncio.CancelledError): stream_task.cancel() try: await stream_task except asyncio.CancelledError: pass except Exception as e: logger.debug("Stream consumer wait before queued message failed: %s", e) _previewed = bool(result.get("response_previewed")) _already_streamed = bool( (_sc and getattr(_sc, "final_response_sent", False)) or _previewed ) first_response = result.get("final_response", "") if first_response and not _already_streamed: try: logger.info( "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.", session_key or "?", ) await adapter.send( source.chat_id, first_response, metadata=_status_thread_metadata, ) except Exception as e: logger.warning("Failed to send first response before queued message: %s", e) elif first_response: logger.info( "Queued follow-up for session %s: skipping resend because final streamed delivery was confirmed.", session_key or "?", ) # Release deferred bg-review notifications now that the # first response has been delivered. Pop from the # adapter's callback dict (prevents double-fire in # base.py's finally block) and call it. if getattr(type(adapter), "pop_post_delivery_callback", None) is not None: _bg_cb = adapter.pop_post_delivery_callback( session_key, generation=run_generation, ) if callable(_bg_cb): try: _bg_cb() except Exception: pass elif adapter and hasattr(adapter, "_post_delivery_callbacks"): _bg_cb = adapter._post_delivery_callbacks.pop(session_key, None) if callable(_bg_cb): try: _bg_cb() except Exception: pass # else: interrupted — discard the interrupted response ("Operation # interrupted." is just noise; the user already knows they sent a # new message). updated_history = result.get("messages", history) next_source = source next_message = pending next_message_id = None next_channel_prompt = None if pending_event is not None: next_source = getattr(pending_event, "source", None) or source next_message = await self._prepare_inbound_message_text( event=pending_event, source=next_source, history=updated_history, ) if next_message is None: return result next_message_id = getattr(pending_event, "message_id", None) next_channel_prompt = getattr(pending_event, "channel_prompt", None) # Restart typing indicator so the user sees activity while # the follow-up turn runs. The outer _process_message_background # typing task is still alive but may be stale. _followup_adapter = self.adapters.get(source.platform) if _followup_adapter: try: await _followup_adapter.send_typing( source.chat_id, metadata=_status_thread_metadata, ) except Exception: pass return await self._run_agent( message=next_message, context_prompt=context_prompt, history=updated_history, source=next_source, session_id=session_id, session_key=session_key, run_generation=run_generation, _interrupt_depth=_interrupt_depth + 1, event_message_id=next_message_id, channel_prompt=next_channel_prompt, ) finally: # Stop progress sender, interrupt monitor, and notification task if progress_task: progress_task.cancel() interrupt_monitor.cancel() _notify_task.cancel() # Wait for stream consumer to finish its final edit if stream_task: try: await asyncio.wait_for(stream_task, timeout=5.0) except (asyncio.TimeoutError, asyncio.CancelledError): stream_task.cancel() try: await stream_task except asyncio.CancelledError: pass # Clean up tracking tracking_task.cancel() if session_key: # Only release the slot if this run's generation still owns # it. A /stop or /new that bumped the generation while we # were unwinding has already installed its own state; this # guard prevents an old run from clobbering it on the way # out. self._release_running_agent_state( session_key, run_generation=run_generation ) if self._draining: self._update_runtime_status("draining") # Wait for cancelled tasks for task in [progress_task, interrupt_monitor, tracking_task, _notify_task]: if task: try: await task except asyncio.CancelledError: pass # If streaming already delivered the response, mark it so the # caller's send() is skipped (avoiding duplicate messages). # BUT: never suppress delivery when the agent failed — the error # message is new content the user hasn't seen, and it must reach # them even if streaming had sent earlier partial output. # # Also never suppress when the final response is "(empty)" — this # means the model failed to produce content after tool calls (common # with mimo-v2-pro, GLM-5, etc.). The stream consumer may have # sent intermediate text ("Let me search for that…") alongside the # tool call, setting already_sent=True, but that text is NOT the # final answer. Suppressing delivery here leaves the user staring # at silence. (#10xxx — "agent stops after web search") _sc = stream_consumer_holder[0] if isinstance(response, dict) and not response.get("failed"): _final = response.get("final_response") or "" _is_empty_sentinel = not _final or _final == "(empty)" _streamed = bool( _sc and getattr(_sc, "final_response_sent", False) ) # response_previewed means the interim_assistant_callback already # sent the final text via the adapter (non-streaming path). _previewed = bool(response.get("response_previewed")) if not _is_empty_sentinel and (_streamed or _previewed): logger.info( "Suppressing normal final send for session %s: final delivery already confirmed (streamed=%s previewed=%s).", session_key or "?", _streamed, _previewed, ) response["already_sent"] = True return response def _start_cron_ticker(stop_event: threading.Event, adapters=None, loop=None, interval: int = 60): """ Background thread that ticks the cron scheduler at a regular interval. Runs inside the gateway process so cronjobs fire automatically without needing a separate `hermes cron daemon` or system cron entry. When ``adapters`` and ``loop`` are provided, passes them through to the cron delivery path so live adapters can be used for E2EE rooms. Also refreshes the channel directory every 5 minutes and prunes the image/audio/document cache + expired ``hermes debug share`` pastes once per hour. """ from cron.scheduler import tick as cron_tick from gateway.platforms.base import cleanup_image_cache, cleanup_document_cache from hermes_cli.debug import _sweep_expired_pastes IMAGE_CACHE_EVERY = 60 # ticks — once per hour at default 60s interval CHANNEL_DIR_EVERY = 5 # ticks — every 5 minutes PASTE_SWEEP_EVERY = 60 # ticks — once per hour CURATOR_EVERY = 60 # ticks — poll hourly (inner gate handles the real cadence) logger.info("Cron ticker started (interval=%ds)", interval) tick_count = 0 while not stop_event.is_set(): try: cron_tick(verbose=False, adapters=adapters, loop=loop) except Exception as e: logger.debug("Cron tick error: %s", e) tick_count += 1 if tick_count % CHANNEL_DIR_EVERY == 0 and adapters: try: from gateway.channel_directory import build_channel_directory if loop is not None: # build_channel_directory is async (Slack web calls), and # this ticker runs in a background thread. Schedule onto # the gateway event loop and wait briefly for completion # so refresh failures are still logged via the except. fut = asyncio.run_coroutine_threadsafe( build_channel_directory(adapters), loop ) fut.result(timeout=30) except Exception as e: logger.debug("Channel directory refresh error: %s", e) if tick_count % IMAGE_CACHE_EVERY == 0: try: removed = cleanup_image_cache(max_age_hours=24) if removed: logger.info("Image cache cleanup: removed %d stale file(s)", removed) except Exception as e: logger.debug("Image cache cleanup error: %s", e) try: removed = cleanup_document_cache(max_age_hours=24) if removed: logger.info("Document cache cleanup: removed %d stale file(s)", removed) except Exception as e: logger.debug("Document cache cleanup error: %s", e) if tick_count % PASTE_SWEEP_EVERY == 0: try: deleted, remaining = _sweep_expired_pastes() if deleted: logger.info( "Paste sweep: deleted %d expired paste(s), %d pending", deleted, remaining, ) except Exception as e: logger.debug("Paste sweep error: %s", e) # Curator — piggy-back on the existing cron ticker so long-running # gateways get weekly skill maintenance without needing restarts. # maybe_run_curator() is internally gated by config.interval_hours # (7 days by default), so CURATOR_EVERY is just the poll rate — the # real work only fires once per config interval. if tick_count % CURATOR_EVERY == 0: try: from agent.curator import maybe_run_curator maybe_run_curator( idle_for_seconds=float("inf"), on_summary=lambda msg: logger.info("curator: %s", msg), ) except Exception as e: logger.debug("Curator tick error: %s", e) stop_event.wait(timeout=interval) logger.info("Cron ticker stopped") async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool: """ Start the gateway and run until interrupted. This is the main entry point for running the gateway. Returns True if the gateway ran successfully, False if it failed to start. A False return causes a non-zero exit code so systemd can auto-restart. Args: config: Optional gateway configuration override. replace: If True, kill any existing gateway instance before starting. Useful for systemd services to avoid restart-loop deadlocks when the previous process hasn't fully exited yet. """ # ── Duplicate-instance guard ────────────────────────────────────── # Prevent two gateways from running under the same HERMES_HOME. # The PID file is scoped to HERMES_HOME, so future multi-profile # setups (each profile using a distinct HERMES_HOME) will naturally # allow concurrent instances without tripping this guard. from gateway.status import ( acquire_gateway_runtime_lock, get_running_pid, get_process_start_time, release_gateway_runtime_lock, remove_pid_file, terminate_pid, ) existing_pid = get_running_pid() if existing_pid is not None and existing_pid != os.getpid(): if replace: existing_start_time = get_process_start_time(existing_pid) logger.info( "Replacing existing gateway instance (PID %d) with --replace.", existing_pid, ) # Record a takeover marker so the target's shutdown handler # recognises its SIGTERM as a planned takeover and exits 0 # (rather than exit 1, which would trigger systemd's # Restart=on-failure and start a flap loop against us). # Best-effort — proceed even if the write fails. try: from gateway.status import write_takeover_marker write_takeover_marker(existing_pid) except Exception as e: logger.debug("Could not write takeover marker: %s", e) try: terminate_pid(existing_pid, force=False) except ProcessLookupError: pass # Already gone except (PermissionError, OSError): logger.error( "Permission denied killing PID %d. Cannot replace.", existing_pid, ) # Marker is scoped to a specific target; clean it up on # give-up so it doesn't grief an unrelated future shutdown. try: from gateway.status import clear_takeover_marker clear_takeover_marker() except Exception: pass return False # Wait up to 10 seconds for the old process to exit for _ in range(20): try: os.kill(existing_pid, 0) time.sleep(0.5) except (ProcessLookupError, PermissionError): break # Process is gone else: # Still alive after 10s — force kill logger.warning( "Old gateway (PID %d) did not exit after SIGTERM, sending SIGKILL.", existing_pid, ) try: terminate_pid(existing_pid, force=True) time.sleep(0.5) except (ProcessLookupError, PermissionError, OSError): pass remove_pid_file() # remove_pid_file() is a no-op when the PID doesn't match. # Force-unlink to cover the old-process-crashed case. try: (get_hermes_home() / "gateway.pid").unlink(missing_ok=True) except Exception: pass # Clean up any takeover marker the old process didn't consume # (e.g. SIGKILL'd before its shutdown handler could read it). try: from gateway.status import clear_takeover_marker clear_takeover_marker() except Exception: pass # Also release all scoped locks left by the old process. # Stopped (Ctrl+Z) processes don't release locks on exit, # leaving stale lock files that block the new gateway from starting. try: from gateway.status import release_all_scoped_locks _released = release_all_scoped_locks( owner_pid=existing_pid, owner_start_time=existing_start_time, ) if _released: logger.info("Released %d stale scoped lock(s) from old gateway.", _released) except Exception: pass else: hermes_home = str(get_hermes_home()) logger.error( "Another gateway instance is already running (PID %d, HERMES_HOME=%s). " "Use 'hermes gateway restart' to replace it, or 'hermes gateway stop' first.", existing_pid, hermes_home, ) print( f"\n❌ Gateway already running (PID {existing_pid}).\n" f" Use 'hermes gateway restart' to replace it,\n" f" or 'hermes gateway stop' to kill it first.\n" f" Or use 'hermes gateway run --replace' to auto-replace.\n" ) return False # Sync bundled skills on gateway start (fast -- skips unchanged) try: from tools.skills_sync import sync_skills sync_skills(quiet=True) except Exception: pass # Centralized logging — agent.log (INFO+), errors.log (WARNING+), # and gateway.log (INFO+, gateway-component records only). # Idempotent, so repeated calls from AIAgent.__init__ won't duplicate. from hermes_logging import setup_logging setup_logging(hermes_home=_hermes_home, mode="gateway") # Optional stderr handler — level driven by -v/-q flags on the CLI. # verbosity=None (-q/--quiet): no stderr output # verbosity=0 (default): WARNING and above # verbosity=1 (-v): INFO and above # verbosity=2+ (-vv/-vvv): DEBUG if verbosity is not None: from agent.redact import RedactingFormatter _stderr_level = {0: logging.WARNING, 1: logging.INFO}.get(verbosity, logging.DEBUG) _stderr_handler = logging.StreamHandler() _stderr_handler.setLevel(_stderr_level) _stderr_handler.setFormatter(RedactingFormatter('%(levelname)s %(name)s: %(message)s')) logging.getLogger().addHandler(_stderr_handler) # Lower root logger level if needed so DEBUG records can reach the handler if _stderr_level < logging.getLogger().level: logging.getLogger().setLevel(_stderr_level) runner = GatewayRunner(config) # Track whether a signal initiated the shutdown (vs. internal request). # When an unexpected SIGTERM kills the gateway, we exit non-zero so # systemd's Restart=on-failure revives the process. systemctl stop # is safe: systemd tracks stop-requested state independently of exit # code, so Restart= never fires for a deliberate stop. _signal_initiated_shutdown = False # Set up signal handlers def shutdown_signal_handler(): nonlocal _signal_initiated_shutdown # Planned --replace takeover check: when a sibling gateway is # taking over via --replace, it wrote a marker naming this PID # before sending SIGTERM. If present, treat the signal as a # planned shutdown and exit 0 so systemd's Restart=on-failure # doesn't revive us (which would flap-fight the replacer when # both services are enabled, e.g. hermes.service + hermes- # gateway.service from pre-rename installs). planned_takeover = False try: from gateway.status import consume_takeover_marker_for_self planned_takeover = consume_takeover_marker_for_self() except Exception as e: logger.debug("Takeover marker check failed: %s", e) if planned_takeover: logger.info( "Received SIGTERM as a planned --replace takeover — exiting cleanly" ) else: _signal_initiated_shutdown = True logger.info("Received SIGTERM/SIGINT — initiating shutdown") # Diagnostic: log all hermes-related processes so we can identify # what triggered the signal (hermes update, hermes gateway restart, # a stale detached subprocess, etc.). try: import subprocess as _sp _ps = _sp.run( ["ps", "aux"], capture_output=True, text=True, timeout=3, ) _hermes_procs = [ line for line in _ps.stdout.splitlines() if ("hermes" in line.lower() or "gateway" in line.lower()) and str(os.getpid()) not in line.split()[1:2] # exclude self ] if _hermes_procs: logger.warning( "Shutdown diagnostic — other hermes processes running:\n %s", "\n ".join(_hermes_procs), ) else: logger.info("Shutdown diagnostic — no other hermes processes found") except Exception: pass asyncio.create_task(runner.stop()) def restart_signal_handler(): runner.request_restart(detached=False, via_service=True) loop = asyncio.get_running_loop() if threading.current_thread() is threading.main_thread(): for sig in (signal.SIGINT, signal.SIGTERM): try: loop.add_signal_handler(sig, shutdown_signal_handler) except NotImplementedError: pass if hasattr(signal, "SIGUSR1"): try: loop.add_signal_handler(signal.SIGUSR1, restart_signal_handler) except NotImplementedError: pass else: logger.info("Skipping signal handlers (not running in main thread).") # Claim the PID file BEFORE bringing up any platform adapters. # This closes the --replace race window: two concurrent `gateway run # --replace` invocations both pass the termination-wait above, but # only the winner of the O_CREAT|O_EXCL race below will ever open # Telegram polling, Discord gateway sockets, etc. The loser exits # cleanly before touching any external service. import atexit from gateway.status import write_pid_file, remove_pid_file, get_running_pid _current_pid = get_running_pid() if _current_pid is not None and _current_pid != os.getpid(): logger.error( "Another gateway instance (PID %d) started during our startup. " "Exiting to avoid double-running.", _current_pid ) return False if not acquire_gateway_runtime_lock(): logger.error( "Gateway runtime lock is already held by another instance. Exiting." ) return False try: write_pid_file() except FileExistsError: release_gateway_runtime_lock() logger.error( "PID file race lost to another gateway instance. Exiting." ) return False atexit.register(remove_pid_file) atexit.register(release_gateway_runtime_lock) # MCP tool discovery — run in an executor so the asyncio event loop # stays responsive even when a configured MCP server is slow or # unreachable. discover_mcp_tools() uses a blocking 120s wait # internally; calling it from the loop thread would freeze platform # heartbeats (Discord shard, Telegram polling) until it returned. # See #16856. try: from tools.mcp_tool import discover_mcp_tools _loop = asyncio.get_running_loop() await _loop.run_in_executor(None, discover_mcp_tools) except Exception as e: logger.debug("MCP tool discovery failed: %s", e) # Start the gateway success = await runner.start() if not success: return False if runner.should_exit_cleanly: if runner.exit_reason: logger.error("Gateway exiting cleanly: %s", runner.exit_reason) return True # Start background cron ticker so scheduled jobs fire automatically. # Pass the event loop so cron delivery can use live adapters (E2EE support). cron_stop = threading.Event() cron_thread = threading.Thread( target=_start_cron_ticker, args=(cron_stop,), kwargs={"adapters": runner.adapters, "loop": asyncio.get_running_loop()}, daemon=True, name="cron-ticker", ) cron_thread.start() # Wait for shutdown await runner.wait_for_shutdown() if runner.should_exit_with_failure: if runner.exit_reason: logger.error("Gateway exiting with failure: %s", runner.exit_reason) return False # Stop cron ticker cleanly cron_stop.set() cron_thread.join(timeout=5) # Close MCP server connections try: from tools.mcp_tool import shutdown_mcp_servers shutdown_mcp_servers() except Exception: pass if runner.exit_code is not None: raise SystemExit(runner.exit_code) # When a signal (SIGTERM/SIGINT) caused the shutdown and it wasn't a # planned restart (/restart, /update, SIGUSR1), exit non-zero so # systemd's Restart=on-failure revives the process. This covers: # - hermes update killing the gateway mid-work # - External kill commands # - WSL2/container runtime sending unexpected signals # systemctl stop is safe: systemd tracks "stop requested" state # independently of exit code, so Restart= never fires for it. if _signal_initiated_shutdown and not runner._restart_requested: logger.info( "Exiting with code 1 (signal-initiated shutdown without restart " "request) so systemd Restart=on-failure can revive the gateway." ) return False # → sys.exit(1) in the caller return True def main(): """CLI entry point for the gateway.""" import argparse parser = argparse.ArgumentParser(description="Hermes Gateway - Multi-platform messaging") parser.add_argument("--config", "-c", help="Path to gateway config file") parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") args = parser.parse_args() config = None if args.config: import yaml with open(args.config, encoding="utf-8") as f: data = yaml.safe_load(f) or {} config = GatewayConfig.from_dict(data) # Run the gateway - exit with code 1 if no platforms connected, # so systemd Restart=on-failure will retry on transient errors (e.g. DNS) success = asyncio.run(start_gateway(config)) if not success: sys.exit(1) if __name__ == "__main__": main()