i dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ej dkZ ddl mZmZddlmZmZddlmZmZmZmZddlmZejeZedz Zd Zd Zd Z d Z!d Z"d Z#dZ$dZ%de&de'fdZ(eGddZ)GddZ*e*Z+ddl,m-Z-m.Z.ddddgdddddd dd!d d"d#d$d%d"d&d d"d'd$d%d(d)gd*d+Z/d,Z0e-j1dd-e/e0d./dS)0a Process Registry -- In-memory registry for managed background processes. Tracks processes spawned via terminal(background=true), providing: - Output buffering (rolling 200KB window) - Status polling and log retrieval - Blocking wait with interrupt support - Process killing - Crash recovery via JSON checkpoint file - Session-scoped tracking for gateway reset protection Background processes execute THROUGH the environment interface -- nothing runs on the host machine unless TERMINAL_ENV=local. For Docker, Singularity, Modal, Daytona, and SSH backends, the command runs inside the sandbox. Usage: from tools.process_registry import process_registry # Spawn a background process (called from terminal_tool) session = process_registry.spawn(env, "pytest -v", task_id="task_123") # Poll for status result = process_registry.poll(session.id) # Block until done result = process_registry.wait(session.id, timeout=300) # Kill it process_registry.kill(session.id) NWindows) _find_shell_sanitize_subprocess_env) dataclassfield)AnyDictListOptional)get_hermes_homezprocesses.jsoni@ i@ secondsreturnctdt|}|dkr|dSt|d\}}|dkr|d|dSt|d\}}|d|dS)Nr<szm zh m)maxintdivmod)rrminssecshourss ;/home/ubuntu/.hermes/hermes-agent/tools/process_registry.pyformat_uptime_shortrMs As7||A2vvwww2JD$ byy!!$!!!!r""KE4  t   c8eZdZUdZeed<eed<dZeed<dZeed<dZe e ed<dZ e e j ed <dZeed <dZe eed <d Zeed <dZeed<dZe e ed<dZeed<eZe ed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZ e ed<dZ!eed<e"e#Z$e%eed<e"dd Z&e ed!<e"dd Z'e ed"<e"dd Z(eed#<e"d d Z)eed$<e"d d Z*eed%<e"dd Z+eed&<e"dd Z,e ed'<e"e-j.Z/e-j.ed(<e"dd Z0e e-j1ed)<e"dd Z2eed*<dS)+ProcessSessionz3A tracked background process with output buffering.idcommandtask_id session_keyNpidprocessenv_refcwd started_atFexited exit_code output_buffermax_output_charsdetachedhost pid_scopewatcher_platformwatcher_chat_idwatcher_user_idwatcher_user_namewatcher_thread_idrwatcher_intervalnotify_on_complete)default_factorywatch_patterns)defaultrepr _watch_hits_watch_suppressed_watch_disabled_watch_last_emit_at_watch_cooldown_until_watch_strike_candidate_watch_consecutive_strikes_lock_reader_thread_pty)3__name__ __module__ __qualname____doc__str__annotations__r&r'r(r rr) subprocessPopenr*rr+r-floatr.boolr/r0MAX_OUTPUT_CHARSr1r2r4r5r6r7r8r9r:r;rlistr=r r@rArBrCrDrErF threadingLockrGrHThreadrIr rr"r"Xs== GGG LLLGSKC#*.GXj& '...GSC#JFD#Ix}###M3,c,,,HdIscOSOSssc$$$$ %d ; ; ;NDI;;;uQU333K333"U15999s999!E%e<<AAA16t%1P1P1PNHY-.PPPd///D#/////r r"ceZdZdZdZdZededefdZde dedd fd Z d e de fd Z ed eede fdZdee dee fdZed edd fdZededefdZ d>dededededede de fdZ d?dedededededede fdZde fdZde ded ed!ed"ef d#Zde fd$Zde fd%Zd&ede fd'Zd&edee fd(Zd@d*Zd&edefd+ZdAd&ed.ed/edefd0Z dBd&ededefd1Z!d&edefd2Z"d&ed3edefd4Z#dCd&ed3edefd5Z$d&edefd6Z%dBdede&fd7Z'dede fd8Z(dede fd9Z)dBdedefd:Z*d;Z+d<Z,defd=Z-d S)DProcessRegistrya$ In-memory registry of running and finished background processes. Thread-safe. Accessed from: - Executor threads (terminal_tool, process tool handlers) - Gateway asyncio loop (watcher tasks, session reset checks) - Cleanup thread (sandbox reaping coordination) )z'bash: cannot set terminal process groupz"bash: no job control in this shellzno job control in this shellz!cannot set terminal process groupz)tcsetattr: Inappropriate ioctl for devicec(i|_i|_tj|_g|_ddl}||_t|_ tj|_ d|_ d|_ d|_d|_dS)Nrr,)_running _finishedrVrWrGpending_watchersqueueQueuecompletion_queueset_completion_consumed_global_watch_lock_global_watch_window_start_global_watch_window_hits_global_watch_tripped_until$_global_watch_suppressed_during_trip)self _queue_mods r__init__zProcessRegistry.__init__s35 46^%% 79 #"""2<2B2B2D2D*-! #,."2"214'./&25(9:111r textrc|dratfdtjDr<dr%tfdtjDz5ProcessRegistry._clean_shell_noise..s,cc%EU1X-ccccccr r)splitanyr[_SHELL_NOISE_SUBSTRINGSpopjoin)rmrss @r_clean_shell_noisez"ProcessRegistry._clean_shell_noises 4   cccc?;bccccc  IIaLLL cccc?;bccccc yyr sessionnew_textNc|jr|jrdS|jrdSg}d}|D]=}|jD]3}||vr-||||}n4>|sdSt j}d}|j5|jri||jkr^|xj t|z c_ |j s7d|_ |xj dz c_ |j tkrd|_d|_d}d} nR|jr|j sd|_ d|_ ||_|t z|_|xjdz c_|j } d|_ d} dddn #1swxYwY| rp|rl|j|j|j|jd|j |j|j|j|j|jd|jdtd t d d dSd |dd } t| dkr | dddz} ||sdS|j|j|j|jd|| | |j|j|j|j|jd dS)uzScan new output for watch patterns and queue notifications. Called from reader threads with new_text being the freshly-read chunk. Per-session rate limit: at most ONE watch-match notification per WATCH_MIN_INTERVAL_SECONDS. Any match arriving inside the cooldown window is dropped and counts as ONE strike for that window. After WATCH_STRIKE_LIMIT consecutive strike windows, watch_patterns is disabled for this session and the session is promoted to notify_on_complete semantics — one notification when the process actually exits, no more mid-process spam. NFTrwatch_disabledz$Watch patterns disabled for process u — z7 consecutive rate-limit windows triggered (min spacing zms). Falling back to notify_on_complete semantics; you'll get exactly one notification when the process exits.) session_idr'r$type suppressedplatformchat_iduser_id user_name thread_idmessageroiz ...(truncated) watch_match) rr'r$rpatternoutputrrrrrr)r=rBr. splitlinesappendrstriptimerGrDrAlenrErFWATCH_STRIKE_LIMITr;rCWATCH_MIN_INTERVAL_SECONDSr@rbputr#r'r$r5r6r7r8r9ry_global_watch_admit) rjr{r| matched_linesmatched_patternlinepatnowshould_disable return_earlyrrs r_check_watch_patternsz%ProcessRegistry._check_watch_patternss% )@  F >  F ''))  D-  $;;!((777&.*-E    Fikk ]% %% % , %w7T1T1T))S-?-??))6 .6:G366!;669=OOO26/6:2)-# 1;#;;:;G627//2+036P0P-##q(##$6 ,-)$ K% %% %% %% %% %% %% %% %% %% %% %% %% %% %% %N   %))")*#*#6&,")"; ' 8&6&6!(!:!(!:LwzLL-LL(BLLL++& F="-.. v;;  ETE]%77F'',,  F !!!*".!&$0.. 2 2 # # sCEE#&E#rc |j5|jrJ||jkr?|j}d|_d|_||_d|_|dkrdddd|d|ddddddd }nd}nd}|jr ||jkr|xjd z c_d }d}ni||jz t kr||_d|_|jt kr$|tz|_|xjd z c_|}d }n|xjd z c_d}d }dddn #1swxYwY||j ||>|j dddd d t dt dtddddddd |S)a1Return True if this watch_match event is allowed through the global breaker. Semantics: - If we're currently in a cooldown period, drop the event and count it. - Otherwise, slide the rolling window and check the global cap. - If the cap is exceeded, trip the breaker for WATCH_GLOBAL_COOLDOWN_SECONDS and emit ONE summary event so the agent/user sees "N notifications were suppressed" instead of getting them individually. - When the cooldown ends, emit a release summary and reset counters. r,rr%watch_overflow_releasedz%Watch-pattern notifications resumed. z1 match event(s) were suppressed during the flood.) rr'r$rrrrrrrrNr~FTwatch_overflow_trippedzWatch-pattern overflow: >z notifications in zCs across all processes. Suppressing further watch_match events for zs.) rr'r$rrrrrrr) rerhrirfrgWATCH_GLOBAL_WINDOW_SECONDSWATCH_GLOBAL_MAX_PER_WINDOWWATCH_GLOBAL_COOLDOWN_SECONDSrbr)rjrr release_msgadmittrip_nows rrz#ProcessRegistry._global_watch_admit;sb $3 !3 !/ #C4;[4[4[!F 360<=925/12.>>')')#% 9&0])]]]%'#%#%%'%'##KK"#'KK" / !C$:Z4Z4Z99Q>9988>h,,71D1DS1I1I7#??3//636 P P P I3OOOOOOOO PvsA A.. B8BBr%Fr$r+r&r'env_varsuse_ptyc tdtjjdd||||pt jt j}|ri trddlm }nddl m }t} ttj |} d| d<|| d d |g|j| d } | j|_| |_t%j|j|fd d|j} | |_| |j5|||j|j<dddn #1swxYwY||S#t8$rt:dYn1t>$r%} t:d| Yd} ~ nd} ~ wwxYwt} ttj |}d|d<tAj!| d d |gd |j|ddt@j"t@j#t@j"trdn tj$ }||_%|j|_t%j|j&|fd d|j} | |_| |j5|||j|j<dddn #1swxYwY||S)aw Spawn a background process locally. Only for TERMINAL_ENV=local. Other backends use spawn_via_env(). Args: use_pty: If True, use a pseudo-terminal via ptyprocess for interactive CLI tools (Codex, Claude Code, Python REPL). Falls back to subprocess.Popen if ptyprocess is not installed. proc_N )r#r$r&r'r+r-r) PtyProcess1PYTHONUNBUFFEREDz-liczset +m; )rx)r+r dimensionsTzproc-pty-reader-targetargsdaemonnamez3ptyprocess not installed, falling back to pipe modez0PTY spawn failed (%s), falling back to pipe modeutf-8replace) rmr+rencodingerrorsstdoutstderrstdin preexec_fnz proc-reader-)'r"uuiduuid4hexrgetcwdrrwinptyr ptyprocessrrenvironspawnr+r(rIrVrX_pty_reader_loopr#rHstartrG_prune_if_neededr]_write_checkpoint ImportErrorrwarningrrPrQPIPESTDOUTsetsidr) _reader_loop)rjr$r+r&r'rrr{_PtyProcessCls user_shellpty_envpty_procreaderebg_envprocs r spawn_localzProcessRegistry.spawn_locals&!.tz||',..#"ry{{y{{     ( V& VHCCCCCCCGGGGGG(]] 22:xHH.1*+)//)=G)=)=> ( 0 'l ' #)0!8GJ88  *0& Z88))+++07DM'*-888888888888888&&((( V V VTUUUUU V V VQSTUUUUUUUU V !]] *"*h??%(!" !5G!5!5 6 ?$/*9tt     h !$, ,,     "(  Z 0 0  ! ! # # #(/DM'* % 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0    sUCE0$E E0EE0EE00$G GF??G)$KK Krtimeoutc tdtjjdd||||t j|d}||}|d|jd} |d|jd} |d|jd } tj|} tj|} tj| }tj| }tj| }d | d | d |d |d|d| } | ||}| dd }| D]@}| }| rt||_nAn/#t $r"}d|_d|_d||_Yd}~nd}~wwxYw|jsEt)j|j||| | | fdd|j}||_||j5|||j|j<dddn #1swxYwY||S)a# Spawn a background process through a non-local environment backend. For Docker/Singularity/Modal/Daytona/SSH: runs the command inside the sandbox using the environment's execute() interface. We wrap the command to capture the in-sandbox PID and redirect output to a log file inside the sandbox, then poll the log via subsequent execute() calls. This is less capable than local spawn (no live stdout pipe, no stdin), but it ensures the command runs in the correct sandbox context. rNrsandbox)r#r$r&r'r+r-r*r4z /hermes_bg_z.logz.pidz.exitz mkdir -p z && ( nohup bash -lc z > z$ 2>&1; rc=$?; printf '%s\n' "$rc" > z ) & echo $! > z && cat rrr%TzFailed to start: z proc-poller-r)r"rrrrrr#shlexquoteexecutegetstriprisdigitrr(rr.r/r0rVrX_env_poller_looprHrrGrr]r)rjrr$r+r&r'rr{rlog_pathpid_path exit_pathquoted_commandquoted_temp_dirquoted_log_pathquoted_pid_pathquoted_exit_path bg_commandresultrrrrs r spawn_via_envzProcessRegistry.spawn_via_env;s.(!.tz||',..#y{{    %%c**;;7:;;;;;7:;;;==GJ=== W--+h//+h//+h// ;y11 D D D . D D3B D D/? D D) D D3B D D  <[[W[==FZZ"--3355F))++  zz||<<>>"%d))GKE < < <!GN "G $;$;$;G ! ! ! ! ! ! < ~ %,sHh B0GJ00 F &,G " LLNNN Z 0 0  ! ! # # #(/DM'* % 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0    s+?BF GF;;G$II  I cd} |jjd}|sn|r||}d}|j5|xj|z c_t |j|jkr|j|j d|_dddn #1swxYwY|||n2#t$r%}t d|Yd}~nd}~wwxYw |j dn2#t$r%}t d|Yd}~nd}~wwxYwd|_ |jj|_||dS# |j dn2#t$r%}t d|Yd}~nd}~wwxYwd|_ |jj|_||wxYw) z:Background thread: read stdout from a local Popen process.TFNzProcess stdout reader ended: %srz$Process wait timed out or failed: %s)r)rreadrzrGr0rr1rrrrwaitr. returncoder/r)rjr{ first_chunkchunkrs rrzProcessRegistry._reader_loops  , ;.33D99( 33E::E"'K]bb))U2))7011G4LLL070EwG_F_F`F`0a-bbbbbbbbbbbbbbb**7E::: ; ? ? ? LL:A > > > > > > > > ? H$$Q$//// H H H CQGGGGGGGG H!GN ' :G   " "7 + + + + +  H$$Q$//// H H H CQGGGGGGGG H!GN ' :G   " "7 + + + +sAB>AB B>BB>"B#B>=E.> C-C(#E.(C--E.1D D<D77D<.G,0F  G, F;F61G,6F;;1G,rrrcNtj|}tj|}tj|}d} |js^tjd |d|dd} | dd} | rt| | kr | | d nd} t| } |j5| |_ t|j |j kr|j |j d |_ d d d n #1swxYwY| r| || |d |d d } | dd }|r| d  dkr|d|dd }|dd } t| d  |_n#t t"f$r d |_YnwxYwd|_||d Sn4#t&$r'd|_d |_||Yd SwxYw|j\d Sd S)zBBackground thread: poll a sandbox log file for non-local backends.rzcat 2>/dev/nullrrrr%Nzkill -0 "$(cat z# 2>/dev/null)" 2>/dev/null; echo $?rr0T)rrr.rsleeprrrrGr0r1rrrrr/ ValueError IndexErrorrr)rjr{rrrrr r r prev_output_lenr new_outputdeltacheck check_output exit_resultexit_strs rrz ProcessRegistry._env_poller_loops> +h//+h// ;y11.+  JqMMM) %IO%I%I%ISUVV#ZZ"55  C>  L$;$;$=$=b$A$G$G$I$IS$P$P"%++=/=== !#.##K +x<<BBDDH/,/0C0C0E0Eb0I0O0O0Q0Q,R,R))& 3///,.)))/%)GN**7333F   !%$&!&&w///  M.+ + + + + s\A(I'?D I'DI'DCI'+>H*)I'*II'II''-JJc|j} |r |d}|rt|tr|n|dd}|j5|xj|z c_t|j|j kr|j|j d|_dddn #1swxYwY| ||n#t$rYn#t$rYnwxYw|n2#t$r%}td|Yd}~nd}~wwxYw |n2#t$r%}td|Yd}~nd}~wwxYwd|_t#|d r|jnd |_||dS) z2Background thread: read output from a PTY process.rrrrNzPTY stdout reader ended: %sz PTY wait timed out or failed: %sT exitstatusr)rIisaliverrrNdecoderGr0rr1rEOFErrorrrrrr.hasattrr(r/r)rjr{ptyrrmrs rrz ProcessRegistry._pty_reader_loops^l ;++--  HHTNNEB(25#(>(>kuuELLQXajLDkDk$]jj#11T911"7#899G2 C >CC CC D C8*D, C85D7C88D D?D::D?E F"FFc|j5|j|jddu}||j|j<dddn #1swxYwY||r_|jrZddlm}|j r||j ddnd}|j d|j|j |j |ddSdSdS)uMove a session from running to finished. Idempotent: if the session was already moved (e.g. kill_process raced with the reader thread), the second call is a no-op — no duplicate completion notification is enqueued. Nr strip_ansi0r% completion)rrr$r/r)rGr]rxr#r^rr;tools.ansi_stripr0r0rbrr$r/)rjr{ was_runningr0 output_tails rrz!ProcessRegistry._move_to_finishedsIZ 1 1-++GJ==TIK)0DN7: & 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1     75  3 3 3 3 3 3GNG\d**W%:566%BCCCbdK  ! % %$%j"?$.% ''         s2AA  A rc||jvS)zJCheck if a completion notification was already consumed via wait/poll/log.)rd)rjrs ris_completion_consumedz&ProcessRegistry.is_completion_consumedsT666r c|j5|j|p|j|}dddn #1swxYwY||S)z*Get a session by ID (running or finished).N)rGr]rr^r)rjrr{s rrzProcessRegistry.gets Z V Vm'' 33Ut~7I7I*7U7UG V V V V V V V V V V V V V V V--g666s5A  A A r"c||jrdSt|dd}|dS |}n#t$rYdSwxYw|dSd}t|dd}|atsY ddl}|}|||j}|||j|tj z | } | r.t| tr| n| dd}n#tt t"f$rYnwxYw |||j|nB#t$rYn6wxYw# |||j|w#t$rYwwxYwxYwn8#t$r+} t$d |j| Yd} ~ nd} ~ wwxYw|j5|rG|xj|z c_t/|j|jkr|j|j d|_d |_||_dddn #1swxYwYt$d |j|||dS) u$Reconcile session.exited against the real child process state. The reader thread (`_reader_loop`) sets `session.exited = True` only in its `finally` block, which runs when `stdout.read()` returns EOF. If the direct `Popen` child has exited but a descendant process (e.g. a daemon spawned by `hermes update` restarting the gateway) is still holding the stdout pipe open, the reader blocks forever and poll() keeps returning "running" indefinitely (issue #17327 — 74 polls over 7 minutes on Feishu). This helper closes that window: when `session.exited` is still False but the direct child's `Popen.poll()` reports an exit code, drain any readable bytes non-blocking and flip `session.exited`. The orphaned reader thread remains stuck on its blocking `read()` but is a daemon thread and will be reaped with the process. Safe no-op on sessions without a local `Popen` (env/PTY), already- exited sessions, and detached-recovered sessions. Nr)r%rrrrr'z$Non-blocking drain failed for %s: %sTzxReconciled session %s: direct child exited with code %s but reader was still blocked (orphaned pipe). Flipped to exited.)r.rpollrrfcntlfilenoF_GETFLF_SETFLr O_NONBLOCKrrrNr*BlockingIOErrorrrrrr#rGr0rr1r/infor) rjr{rrcdrainedrr;fdflagsrrs r_reconcile_local_exitz%ProcessRegistry._reconcile_local_exit#s!( ?gn? Fw 400 < F BB    FF  : F x..  k  T ]]__ B 66 B ur}/DEEE "KKMMEo+5eS+A+A"n%%u||T[dm|GnGn'*=D B u====$ B u====$ T T T CWZQRSSSSSSSS T] # # ^%%0%%w,--0HHH,3,A7C[B[B\B\,]G)!GN "G   # # # # # # # # # # # # # # #   D J   w'''''s7 AA+AF AD E D'$E&D''E+EF  EF EF FE76F7 FFFFF G!F;;G AH..H25H2cddlm}||}|dd|dS|||j5|jr||jddnd}dddn #1swxYwY|j|j|jrd nd |j ttj |j z |d }|jr$|j |d <|j||jr d |d<d|d<|S)z9Check status and get new output for a background process.rr/N not_foundNo process with ID statuserrorr%r.running)rr$rKr(uptime_secondsoutput_previewr/Tr2z=Process recovered after restart -- output history unavailablenote)r3r0rrFrGr0r#r$r.r(rrr-r/rdaddr2)rjrr0r{rPrs rr:zProcessRegistry.pollks//////((:&& ?)4V*4V4VWW W ""7+++ ] h hJQJ_gZZ(=eff(EFFFegN h h h h h h h h h h h h h h h"*").?hhi;!$)++0B"BCC,    > 6")"3F;   % ) )* 5 5 5   ]!%F: \F6N s"A00A47A4roffsetlimitcddlm}||}|dd|dS|j5||j}dddn #1swxYwY|}t |}|dkr|dkr || d} n ||||z} |j|jrdndd | |t | d d } |jr|j || S) z;Read the full output log with optional pagination by lines.rr/NrHrIrJr.rNroz lines)rrKr total_linesshowing) r3r0rrGr0rrr#r.ryrdrR) rjrrTrUr0r{ full_outputrsrWselectedrs rread_logzProcessRegistry.read_logsr//////((:&& ?)4V*4V4VWW W ] < <$*W%:;;K < < < < < < < < < < < < < < <&&((%jj  Q;;5199eVWW~HHVFUN23H"*").?hhiii))&h--///    > 6  % ) )* 5 5 5 sA  AAcbddlm}ddlm} t t jdd}n#ttf$rd}YnwxYw|}|}d}|r||kr |} d|d |d }n|p|} | |} | d d |d Stj | z} tj | kr| | } | | | jrD|j|d| j|| jddd} |r|| d<| S|r%d|| jdddd} |r|| d<| Stjdtj | kd|| jddd} |r|| d<n d| d| d<| S)aY Block until a process exits, timeout, or interrupt. Args: session_id: The process to wait for. timeout: Max seconds to block. Falls back to TERMINAL_TIMEOUT config. Returns: dict with status ("exited", "timeout", "interrupted", "not_found") and output snapshot. rr/)is_interruptedTERMINAL_TIMEOUT180NzRequested wait of z%s was clamped to configured limit of rrHrIrJr.r1)rKr/r timeout_note interruptedrMz+User sent a new message -- wait interrupted)rKrrQr~r)rKrzWaited zs, process still running)r3r0tools.interruptr]rrgetenvr TypeErrorrr monotonicrrFr.rdrRr/r0r) rjrrr0_is_interrupteddefault_timeout max_timeoutrequested_timeoutraeffective_timeoutr{deadliners rrzProcessRegistry.waits 0/////EEEEEE "!"),>"F"FGGOOI& " " "!OOO "% #  A!2[!@!@ + 9%699*5999 L !2 @[ ((:&& ?)4V*4V4VWW W>##&77n))44W==G  & &w / / /~ )--j999&!(!2(j)>uvv)FGG  :-9F>*    +(j)>uvv)FGGI  :-9F>* JqMMM7n))<  j!6uvv!>??    [%1F> " "%Z/@%Z%Z%ZF> " s"1AAc||}|dd|dS|jr d|jdS |jrZ |jdn#t $r/|jr$tj|jtj YnmwxYw|j r tr|j n;tj tj|j jtj n#tt f$r|j YnwxYw|jr-|jr&|jd |jd d n|jr|jd kr|jr||jsL|j5d|_d|_dddn #1swxYwY||d|jdS||jndddSd|_d|_|||d|jdS#t $r}dt7|dcYd}~Sd}~wwxYw)zKill a background process.NrHrIrJalready_exited)rKr/T)forcezkill rrrr3rLzkRecovered process cannot be killed after restart because its original runtime handle is no longer availableikilled)rKr)rr.r/rI terminaterr(rrrrr)rrrrrr*rr2r4rrGrrrr#rNrjrr{rs r kill_processzProcessRegistry.kill_processs((:&& ?)4V*4V4VWW W > *$.  - 8|% =L***6666 ==={= V^<<<= +"S113333 "*W_-@"A"A6>RRR*O<+++O((*****+ W[ ''(I (I(I(IST'UUUU! g&76&A&Agk&A..w{;; 11)-,0)111111111111111**7333"2%,%6((5555&M"GN #G   " "7 + + +  " " $ $ $&gjAA A 8 8 8%A77 7 7 7 7 7 7 8sH=AH=5B H=B H=AC97H=9*D&#H=%D&&A1H=F2& H=2F66H=9F6:!H=H= Y.9WXX X 7F # # <  < <3=dC3H3HR4;;w///d ""8,,,"&TCCC < < <")CFF;;;;;;;; < sgo&; s%0qrr r 8 O ! ' ' - - - O ! ' ' ) ) )"SYY?? ? 8 8 8%A77 7 7 7 7 7 7 8s>AB C&B=7C=CAD-- E7EEEc4|||dzS)zGSend data + newline to a running process's stdin (like pressing Enter).ro)r~)rjrrts r submit_stdinzProcessRegistry.submit_stdinMs D4K888r c||}|dd|dS|jrdddSt|drO|jrH |jdd d S#t $r}d t |dcYd}~Sd}~wwxYw|jr |jjsd d dS |jj dd d S#t $r}d t |dcYd}~Sd}~wwxYw)zGClose a running process's stdin / send EOF without killing the process.NrHrIrJrnrvrIrwzEOF sent)rKrrLryz stdin closed) rr.r,rIsendeofrrNr)rcloserrs r close_stdinzProcessRegistry.close_stdinQsb((:&& ?)4V*4V4VWW W > Y.9WXX X 7F # # <  < < $$&&&"&:>>> < < <")CFF;;;;;;;; < sgo&; s%0qrr r 8 O ! ' ' ) ) )"~>> > 8 8 8%A77 7 7 7 7 7 7 8s<A"" B,B=BB$"C C-C("C-(C-c j5tjtjz}dddn #1swxYwYfd|D}rfd|D}g}|D]}|j|jdd|j|jtj dtj |j ttj |j z |jrdnd|jr|jddnd d }|jr |j|d <|jrd |d <|||S)z1List all running and recently-finished processes.Nc:g|]}|SrY)r)rqrrjs r z1ProcessRegistry.list_sessions..ms'PPPa66q99PPPr c*g|]}|jk |SrYr&rqrr&s rrz1ProcessRegistry.list_sessions..ps%LLL!qyG7K7KA7K7K7Kr rSz%Y-%m-%dT%H:%M:%Sr.rNi8r%)rr$r+r(r-rOrKrPr/Tr2)rGrUr]valuesr^r#r$r+r(rstrftime localtimer-rr.r0r/r2r)rjr& all_sessionsrrentrys`` r list_sessionszProcessRegistry.list_sessionshs Z X X 4 4 6 677$t~?T?T?V?V:W:WWL X X X X X X X X X X X X X X XQPPP<PPP  MLLLL|LLLL ! !Ad9TcT?uu"m,?PQP\A]A]^^"%dikkAL&@"A"A&'h=((I<=O"S!/$%%"8"8QS  Ex 1%&[k"z )$(j! MM%  sAA$$A(+A(ch|j5t|j}dddn #1swxYwY|D]}|||j5t fd|jDcdddS#1swxYwYdS)zK|]}|jko|j VdSNr&r.rs rrtz7ProcessRegistry.has_active_processes..sG W$5QXr rGrUr]rrrv)rjr&sessionsr{s ` rhas_active_processesz$ProcessRegistry.has_active_processesP Z 4 4DM002233H 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4  4 4G  * *7 3 3 3 3 Z  --//                  !'<AA(2B''B+.B+ch|j5t|j}dddn #1swxYwY|D]}|||j5t fd|jDcdddS#1swxYwYdS)z>Check if there are active processes for a gateway session key.Nc3>K|]}|jko|j VdSr)r'r.)rqrr's rrtz9ProcessRegistry.has_active_for_session..sG ,=QXr r)rjr'rr{s ` rhas_active_for_sessionz&ProcessRegistry.has_active_for_sessionrrc |j5fd|jD}dddn #1swxYwYd}|D]8}||j}|ddvr|dz }9|S)zQKill all running processes, optionally filtered by task_id. Returns count killed.c<g|]} |jk|j|Srrrs rrz,ProcessRegistry.kill_all..s;OqyG';';QX';';';';r NrrK)rprnr~)rGr]rrsr#r)rjr&targetsrpr{rs ` rkill_allzProcessRegistry.kill_alls Z  =//11G                  G&&wz22Fzz(##'CCC!  s &;??cVtjfdjD}|D]$}j|=j|%t jt jz}|tkrBjr;tjfd}j|=j|j j z}j|z }|rxj|zc_dSdS)zGRemove oldest finished sessions if over MAX_PROCESSES. Must hold _lock.c@g|]\}}|jz tk|SrY)r-FINISHED_TTL_SECONDS)rqsidrrs rrz4ProcessRegistry._prune_if_needed..s:   Cal"&::: :::r c(j|jSr)r^r-)rrjs rz2ProcessRegistry._prune_if_needed..sDN3)>>)G3  /  % % . % % % % / /r cT |j5g}|jD]}|js|id|jd|jd|jd|jd|j d|j d|j d|j d |j d |jd |jd |jd |jd|jd|jd|j dddn #1swxYwYddlm}|t.|dS#t0$r(}t2d|dYd}~dSd}~wwxYw)z=Write running process metadata to checkpoint file atomically.rr$r(r4r+r-r&r'r5r6r7r8r9r:r;r=Nr)atomic_json_writez#Failed to write checkpoint file: %sT)exc_info)rGr]rr.rr#r$r(r4r+r-r&r'r5r6r7r8r9r:r;r=utilsrCHECKPOINT_PATHrrr)rjentriesrrrs rrz!ProcessRegistry._write_checkpoints R  --//A8((!$(%qy("15(( ( "15 ( )!, (&qy(*1=(/0B(.q/@(.q/@(01D(01D(/0B(1!2F( -a.>!(               0 0 / / / / /  ow 7 7 7 7 7 R R R LL>DL Q Q Q Q Q Q Q Q Q Rs;C5B;C C5CC5CC55 D'?D""D'c  tsdS tjtd}n#t $rYdSwxYwd}|D]}|d}|s|dd}|dkr:td|dd d d ||q| |}|rtd!id |d d|dd d|ddd|ddd|d|d|dd|dtj ddd|ddd|ddd|ddd|ddd|ddd|ddd|ddd|dg}|j 5||j |j<d d d n #1swxYwY|dz }td|jd d ||jdkrQ|j|j|j|j|j|j|j|j|j|jd ||S)"z On gateway startup, probe PIDs from checkpoint file. Returns the number of processes recovered as detached. rr)rr(r4r3z=Skipping recovery for non-host process: %s (pid=%s, scope=%s)r$unknownNrr#rr&r%r'r+r-r2Tr5r6r7r8r9r:r;Fr=r~z'Recovered detached process: %s (pid=%d)) rcheck_intervalr'rrrrrr;rY)rexistsjsonloads read_textrrrrArr"rrGr]r#r$r:r_rr'r5r6r7r8r9r;r)rjr recoveredrr(r4aliver{s rrecover_from_checkpointz'ProcessRegistry.recover_from_checkpoints %%'' 1 j!:!:G!:!L!LMMGG   11  : : E))E""C  +v66IF"" SIIi33CRC8  ++C00E% (\**!IIi;;;"IIi444!& - < < <    (i  %((( %yyty{{CCC"T&+YY/A2%F%F%F%*II.?$D$D$D%*II.?$D$D$D',ii0CR&H&H&H',ii0CR&H&H&H&+YY/A1%E%E%E (-yy1Eu'M'M'M!"$)99-=r#B#B#B#&Z8807DM'*-888888888888888Q  EwWZXZWZG[]`aaa+a//)00&-j*1*B'.':$+$<#*#:#*#:%,%>%,%>.5.H 2 2       s#-A AAI##I' *I' )Nr%r%NF)Nr%r%r)r{r"rN)rrSr)r%).rJrKrLrMrwrl staticmethodrNrzr"rrRrSrr rrrrrrdictrrrrrrr7rrFr:r[rrsr~rrrUrrrrrrrrYr rr[r[s[;;;8     \ | ^| s| t| | | | |UuUUUUUn $\.1IhWeNf& ) ) ) ) )\ ) 3 3   \ nnnn n  n  nn nnnnhNN NN N  N  NN NNNNd,N,,,,83%3,/3;>3JM3Z]3333j(((((>87777777c7h~&>7777 F(F(F(F(Pst>3d>KKsKSKDKKKKZ:8s:8t:8:8:8:8x8c888888699s9#9t99998c8d8888.SD> C D     # $       s    "///:RRR@MMMMMMMr r[)registry tool_errorr)afManage background processes started with terminal(background=true). Actions: 'list' (show all), 'poll' (check status + new output), 'log' (full output with pagination), 'wait' (block until done or timeout), 'kill' (terminate), 'write' (send raw stdin data without newline), 'submit' (send data + Enter, for answering prompts), 'close' (close stdin/send EOF).objectstring)rUr:logrrr{submitrz)Action to perform on background processes)renum descriptionz]Process session ID (from terminal background output). Required for all actions except 'list'.)rrz@Text to send to process stdin (for 'write' and 'submit' actions)integerzJMax seconds to block for 'wait' action. Returns partial output on timeout.r~)rrminimumz6Line offset for 'log' action (default: last 200 lines)z$Max lines to return for 'log' action)actionrrtrrTrUr)r propertiesrequired)rr parametersc r|d}|dd}|d#t|ddnd}|dkr1tjdt|id S|d vr|st d |S|d kr.tjt|d S|d krYtjt||dd|ddd S|dkrCtjt ||dd S|dkr.tjt |d S|dkrPtjt |t|ddd S|dkrPtjt |t|ddd S|dkr.tjt |d St d|dS)Nr&rr%rrU processesrF) ensure_ascii)r:rrrr{rrzsession_id is required for r:rrTrrUrS)rTrUrrrrr{rtrrzUnknown process action: z8. Use: list, poll, log, wait, kill, write, submit, close)rrNrdumpsprocess_registryrrr:r[rrsr~rr)rkwr&rrs r_handle_processrwsffYG XXh # #F48HH\4J4J4VTXXlB//000\^J z;(8(F(Fw(F(W(WXglmmmm N N N FDFDDEE E V  :.33J??eTTT T u__:.77488Ha#8#8RU@V@V8XXfkmmm m v  :.33JQZH[H[3\\kpqqq q v  :.;;JGGV[\\\ \ w  :.:::s488TZ\^K_K_G`G`aapuvvv v x  :.;;JDHHU[]_L`L`HaHabbqvwww w w  :.:::FFUZ[[[ [ qqqq r rrr terminalu⚙️)rtoolsetschemahandleremoji)2rMrloggingrrrrrPrVrrsystemrtools.environments.localrr dataclassesrrtypingrr r r hermes_cli.configr getLoggerrJrrrTrrrrrrrrrNrr"r[rtools.registryrrPROCESS_SCHEMArregisterrYr rrs>    ho9, JJJJJJJJ((((((((,,,,,,,,,,,,------  8 $ $"/##&66  !  " +0+0+0+0+0+0+0 +0\wwwwwwwwv%#?$$ 0///////  _![[[J !~ !a "k "W "E/  :J?  **Zsss:      r