
    iMA                     l   d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
	 ddlZn# e$ r dZY nw xY wddlmZ ddlmZ ddlmZ ddlmZ  ej        e          Ze
j        ZdZdZeeegdf         Zeeeeef                  gdf         Z eegdf         Z!eee         gdf         Z"eg eeeef                  f         Z#dd
edeeeef                  fdZ$dee         defdZ%dee         defdZ&deeeef                  dee         fdZ'dedefdZ(dZ)dZ*dZ+ G d d          Z,dS )a  Shared file sync manager for remote execution backends.

Tracks local file changes via mtime+size, detects deletions, and
syncs to remote environments transactionally.  Used by SSH, Modal,
and Daytona.  Docker and Singularity use bind mounts (live host FS
view) and don't need this.
    N)Path)Callable)get_hermes_home)_file_mtime_keyg      @HERMES_FORCE_FILE_SYNC/root/.hermescontainer_basereturnc                 t   ddl m}m}m} g } |            D ]<}|d                             d| d          }|                    |d         |f           = ||           D ]%}|                    |d         |d         f           & ||           D ]%}|                    |d         |d         f           &|S )a[  Enumerate all files that should be synced to a remote environment.

    Combines credentials, skills, and cache into a single flat list of
    (host_path, remote_path) pairs.  Credential paths are remapped from
    the hardcoded /root/.hermes to *container_base* because the remote
    user's home may differ (e.g. /home/daytona, /home/user).
    r   )get_credential_file_mountsiter_cache_filesiter_skills_filescontainer_pathr      	host_path)r	   )tools.credential_filesr   r   r   replaceappend)r	   r   r   r   filesentryremotes          A/home/ubuntu/.hermes/hermes-agent/tools/environments/file_sync.pyiter_sync_filesr   1   s#             $&E++-- 3 3'(00^Q
 
 	eK(&12222"".AAA D DeK(%0@*ABCCCC!!@@@ D DeK(%0@*ABCCCCL    remote_pathsc                 F    dd                     d | D                       z   S )z<Build a shell ``rm -f`` command for a batch of remote paths.zrm -f  c              3   >   K   | ]}t          j        |          V  d S Nshlexquote).0ps     r   	<genexpr>z$quoted_rm_command.<locals>.<genexpr>P   s*      DD!u{1~~DDDDDDr   join)r   s    r   quoted_rm_commandr(   N   s(    chhDD|DDDDDDDr   dirsc                 F    dd                     d | D                       z   S )z>Build a shell ``mkdir -p`` command for a batch of directories.z	mkdir -p r   c              3   >   K   | ]}t          j        |          V  d S r   r    )r#   ds     r   r%   z'quoted_mkdir_command.<locals>.<genexpr>U   s*      !?!?Q%+a..!?!?!?!?!?!?r   r&   )r)   s    r   quoted_mkdir_commandr-   S   s(    !?!?$!?!?!?????r   r   c                 4    t          d | D                       S )zCExtract sorted unique parent directories from (host, remote) pairs.c                 V    h | ]&\  }}t          t          |          j                  'S  )strr   parentr#   _r   s      r   	<setcomp>z%unique_parent_dirs.<locals>.<setcomp>Z   s-    CCC	63tF||*++CCCr   )sorted)r   s    r   unique_parent_dirsr7   X   s    CCUCCCDDDr   pathc                     t          j                    }t          | d          5 t          fdd          D ]}|                    |           	 ddd           n# 1 swxY w Y   |                                S )z$Return hex SHA-256 digest of a file.rbc                  .                          d          S )Ni   )read)fs   r   <lambda>z_sha256_file.<locals>.<lambda>a   s    !&&-- r   r   N)hashlibsha256openiterupdate	hexdigest)r8   hchunkr=   s      @r   _sha256_filerG   ]   s    A	dD		 Q////55 	 	EHHUOOOO	               ;;==s   ,AA"%A"   )         l        c                      e Zd ZdZeddfdedededede	dz  de
dz  fd	Zd
ddeddfdZddedz  ddfdZdeddfdZdeddfdZddZ	 ddedeeeef                  dz  dedz  fdZ	 ddedeeeef                  dz  dedz  fdZdS )FileSyncManageru  Tracks local file changes and syncs to a remote environment.

    Backends instantiate this with transport callbacks (upload, delete)
    and a file-source callable.  The manager handles mtime-based change
    detection, deletion tracking, rate limiting, and transactional state.

    Not used by bind-mount backends (Docker, Singularity) — those get
    live host FS views and don't need file sync.
    Nget_files_fn	upload_fn	delete_fnsync_intervalbulk_upload_fnbulk_download_fnc                     || _         || _        || _        || _        || _        i | _        i | _        d| _        || _        d S )Ng        )	_get_files_fn
_upload_fn_bulk_upload_fn_bulk_download_fn
_delete_fn_synced_files_pushed_hashes_last_sync_time_sync_interval)selfrN   rO   rP   rQ   rR   rS   s          r   __init__zFileSyncManager.__init__v   sP     *#-!1#;=.0&)+r   F)forcer`   r
   c                   |sLt           j                            t                    s(t	          j                    }|| j        z
  | j        k     rdS |                                 }d |D             g }t          | j
                  }|D ]R\  }}t          |          }|| j
                            |          |k    r6|                    ||f           |||<   Sfd| j
        D             }	|s|	st	          j                    | _        dS t          | j
                  }
t          | j                  }|r(t                              dt!          |                     |	r(t                              dt!          |	                     	 |rE| j        >|                     |           t                              dt!          |                     n:|D ]7\  }}|                     ||           t                              d||           8|	r0|                     |	           t                              d|	           |D ]\  }}t)          |          | j        |<   |	D ]3}|                    |d           | j                            |d           4|| _
        t	          j                    | _        dS # t,          $ rL}|
| _
        || _        t	          j                    | _        t                              d	|           Y d}~dS d}~ww xY w)
aT  Run a sync cycle: upload changed files, delete removed files.

        Rate-limited to once per ``sync_interval`` unless *force* is True
        or ``HERMES_FORCE_FILE_SYNC=1`` is set.

        Transactional: state only committed if ALL operations succeed.
        On failure, state rolls back so the next cycle retries everything.
        Nc                     h | ]\  }}|S r0   r0   r3   s      r   r5   z'FileSyncManager.sync.<locals>.<setcomp>   s    FFF91fFFFr   c                     g | ]}|v|	S r0   r0   )r#   r$   current_remote_pathss     r   
<listcomp>z(FileSyncManager.sync.<locals>.<listcomp>   s$    TTT1a?S6S6SQ6S6S6Sr   zfile_sync: uploading %d file(s)z+file_sync: deleting %d stale remote file(s)z#file_sync: bulk-uploaded %d file(s)zfile_sync: uploaded %s -> %szfile_sync: deleted %sz-file_sync: sync failed, rolled back state: %s)osenvironget_FORCE_SYNC_ENVtime	monotonicr\   r]   rU   dictrZ   r   r   r[   loggerdebuglenrW   rV   rY   rG   pop	Exceptionwarning)r^   r`   nowcurrent_files	to_upload	new_filesr   remote_pathfile_key	to_delete
prev_filesprev_hashesr$   excrd   s                 @r   synczFileSyncManager.sync   sV     	RZ^^O<< 	.""CT))D,???**,,FFFFF ,.	+,,	&3 	. 	."I{&y11H!%%k22h>>i5666%-Ik"" UTTT 2TTT	 	 	#'>#3#3D F $,--
4.// 	LLL:C	NNKKK 	XLLFIWWW	Q YT1=$$Y///BC	NNSSSS.7 Y Y*I{OOI{;;;LL!?KXXXX A	***4i@@@ +4 K K&	;3?	3J3J#K00 1 1a&&&#''40000!*D#'>#3#3D    	Q 	Q 	Q!+D"-D#'>#3#3D NNJCPPPPPPPPP		Qs   	D'J2 2
L<ALLhermes_homec                 4   | j         dS | j        s#| j        st                              d           dS |pt                      dz  }|j                            dd           d}t          t                    D ]}}	 | 
                    |            dS # t          $ rV}|}|t          dz
  k     r<t          |         }t                              d|dz   ||           t          |           Y d}~vd}~ww xY wt                              dt          |           dS )	a  Pull remote changes back to the host filesystem.

        Downloads the remote ``.hermes/`` directory as a tar archive,
        unpacks it, and applies only files that differ from what was
        originally pushed (based on SHA-256 content hashes).

        Protected against SIGINT (defers the signal until complete) and
        serialized across concurrent gateway sandboxes via file lock.
        Nu+   sync_back: no prior push state — skippingz
.sync.lockT)parentsexist_okr   z2sync_back: attempt %d failed (%s), retrying in %dsz%sync_back: all %d attempts failed: %s)rX   r[   rZ   rm   rn   r   r2   mkdirrange_SYNC_BACK_MAX_RETRIES_sync_back_oncerq   _SYNC_BACK_BACKOFFrr   _sleep)r^   r~   	lock_pathlast_excattemptr|   delays          r   	sync_backzFileSyncManager.sync_back   sE    !)F
 " 	4+= 	LLFGGGF 5O$5$5E	td;;;%)344 	" 	"G"$$Y/// " " "3a777.w7ENNL!S%   5MMM" 	>@VX`aaaaas   <B
C4AC//C4r   c                    t          j                    t          j                    u }g d}|rBt          j        t          j                  }fd}t          j        t          j        |           	 |                     |           |rU|Ut          j        t          j        |           r6t          j        t          j	                    t          j                   dS dS dS dS # |rT|St          j        t          j        |           r3t          j        t          j	                    t          j                   w w w w xY w)z>Single sync-back attempt with SIGINT protection and file lock.Nc                 j                         | |f           t                              d           d S )Nz/sync_back: SIGINT deferred until sync completes)r   rm   rn   )signumframedeferred_sigints     r   _defer_sigintz6FileSyncManager._sync_back_once.<locals>._defer_sigint  s4    &&777NOOOOOr   )
	threadingcurrent_threadmain_threadsignal	getsignalSIGINT_sync_back_lockedrf   killgetpid)r^   r   on_main_threadoriginal_handlerr   r   s        @r   r   zFileSyncManager._sync_back_once   sT   
 #133y7L7N7NN(* 	8%/>>P P P P P M&-777	8""9--- 8"2">fm-=>>>" 8GBIKK777778 8">">8 8  8"2">fm-=>>>" 8GBIKK77778">8s   1C# #AD=c                    t           |                                  dS t          |d          }	 t          j        |t           j                   |                                  t          j        |t           j                   |                                 dS # t          j        |t           j                   |                                 w xY w)z;Sync-back under file lock (serializes concurrent gateways).Nw)fcntl_sync_back_implrA   flockLOCK_EXLOCK_UNclose)r^   r   lock_fds      r   r   z!FileSyncManager._sync_back_locked  s    =  """Fy#&&	K///  """K///MMOOOOO K///MMOOOOs   3B 5Cc           	         | j         t          d          	 t          |                                           }n# t          $ r g }Y nw xY wt          j        d          5 }|                      t          |j                             	 t          j
                            |j                  }n# t          $ r d}Y nw xY w|t          k    r/t                              d|t                     	 ddd           dS t          j        d          5 }t#          j        |j                  5 }|                    |d	
           ddd           n# 1 swxY w Y   d}t          j        |          D ]r\  }}}	|	D ]g}
t          j
                            ||
          }t          j
                            ||          }d|z   }| j                            |          }|t3          |          }||k    rznd}|                     ||          }|4|                     ||          }|t                              d|           t          j
                            |          r2|0t3          |          }||k    rt                              d|           t          j        t          j
                            |          d           tA          j!        ||           |dz  }it|rt          "                    d|           nt                              d           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z1Download, diff, and apply remote changes to host.Nz/_sync_back_impl called without bulk_download_fnz.tar)suffixr   uB   sync_back: remote tar is %d bytes (cap %d) — skipping extractionzhermes-sync-back-)prefixdata)filter/z(sync_back: skipping %s (no host mapping)uw   sync_back: conflict on %s — host modified since push, remote also changed. Applying remote version (last-write-wins).T)r   r   z%sync_back: applied %d changed file(s)z%sync_back: no remote changes detected)#rX   RuntimeErrorlistrU   rq   tempfileNamedTemporaryFiler   namerf   r8   getsizeOSError_SYNC_BACK_MAX_BYTESrm   rr   TemporaryDirectorytarfilerA   
extractallwalkr'   relpathr[   rh   rG   _resolve_host_path_infer_host_pathrn   existsmakedirsdirnameshutilcopy2info)r^   file_mappingtftar_sizestagingtarapplieddirpath	_dirnames	filenamesfnamestaged_filerelrw   pushed_hashremote_hashr   	host_hashs                     r   r   zFileSyncManager._sync_back_impl'  s   !)PQQQ	 2 2 4 455LL 	 	 	LLL	 (777 A	J2""4==1117??2733   ...X2   A	J A	J A	J A	J A	J A	J A	J A	J  ,4GHHH 1JG\"'** ;cNN76N:::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; 57WW5E5E '% '%1GY	!* &% &%&(gll7E&B&B gook7CC&)Ci&*&9&=&=k&J&J '2*6{*C*CK*k99 (  : +/K %)$;$;K$V$V	$,(,(=(=k<(X(XI(0 &$N$/!" !" !" !)7>>)44 "9P(4Y(?(?I(K77 &%H %0	!" !" !" BGOOI$>$>NNNN[)<<<1M&%P  JKK GQQQQLL!HIIIc1J 1J 1J 1J 1J 1J 1J 1J 1J 1J 1J 1J 1J 1J 1J!A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	J A	Js   !: A	A	!(L?
$B/.L?/B>;L?=B>>0L?;L?L'*EL'EL'EGL'L?'L+	+L?.L+	/L??MMrw   r   c                 6    ||ng }|D ]\  }}||k    r|c S dS )zAFind the host path for a known remote path from the file mapping.Nr0   )r^   rw   r   mappinghostr   s         r   r   z"FileSyncManager._resolve_host_pathu  sC     #/":,,# 	 	LD&$$ %tr   c                    ||ng }|D ]}\  }}t          t          |          j                  }|                    |dz             r?t          t          |          j                  }|t	          |          d         }||z   c S ~dS )u  Infer a host path for a new remote file by matching path prefixes.

        Uses the existing file mapping to find a remote->host directory
        pair, then applies the same prefix substitution to the new file.
        For example, if the mapping has ``/root/.hermes/skills/a.md`` →
        ``~/.hermes/skills/a.md``, a new remote file at
        ``/root/.hermes/skills/b.md`` maps to ``~/.hermes/skills/b.md``.
        Nr   )r1   r   r2   
startswithro   )	r^   rw   r   r   r   r   
remote_dirhost_dirr   s	            r   r   z FileSyncManager._infer_host_path~  s     #/":,,# 	) 	)LD&T&\\011J%%j3&677 )tDzz011$S__%5%56&(((() tr   r   )r
   N)__name__
__module____qualname____doc___SYNC_INTERVAL_SECONDS
GetFilesFnUploadFnDeleteFnfloatBulkUploadFnBulkDownloadFnr_   boolr}   r   r   r   r   r   r1   r   tupler   r   r0   r   r   rM   rM   k   s          6.226, , , , 	,
 , %t+, )4/, , , ,& %* IQ IQ IQT IQd IQ IQ IQ IQ^&b &bTD[ &bD &b &b &b &bP8 8$ 8 8 8 824 D    LJ LJ LJ LJ^ IM c )-eCHo)>)EQTW[Q[    GK C '+E#s(O'<t'CORUYz     r   rM   )r   )-r   r?   loggingrf   r!   r   r   r   r   r   rj   r   ImportErrorpathlibr   typingr   hermes_constantsr   tools.environments.baser   	getLoggerr   rm   sleepr   r   ri   r1   r   r   r   r   r   r   r   r   r(   r-   r7   rG   r   r   r   rM   r0   r   r   <module>r      s      				          LLLL   EEE             , , , , , , 3 3 3 3 3 3		8	$	$ 
 * S#J$%eCHo./564&$,'T#YK%&b$uS#X//0
 C d5c?>S    :EDI E# E E E E
@tCy @S @ @ @ @
Ed5c?3 ES	 E E E E
s s       - d d d d d d d d d ds   1 ;;