+
    5mj6              
          R t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt]P                  P                  ^ ]P                  P                  ]P                  P                  ]P                  P                  ]4      4      RR4      4       ^ RIHt ^[t]P$                  ! R4      tR tR tR tR	 tRR
 ltR tR tR tR tR tR t]R8X  d   ]P@                  ! ]! 4       4       R# R# )uq  Turn silent multi-hour stalls into immediate loud failures.

Liveness is proven by PROGRESS, never by process existence: a crash-looping or
epoll-blocked child holds a live PID for its whole timeout while doing nothing.
Four orthogonal signals are sampled across the whole process tree. A run is
stuck when EVERY one of them is flat for the full idle window, or when output
alone is flat for the (much longer) output ceiling — the ceiling exists because
ANDing the signals lets one spinning signal veto detection forever.

  output   bytes written to the captured log
  cpu      utime+stime jiffies, recursive
  io       rchar+wchar, recursive (counts socket traffic; read_bytes does not)
  beat     mtime of a heartbeat file a legitimate waiter touches

Subcommands:
  run     supervise a command
  replay  re-run the detector over recorded traces (false-positive verification)
Nz..lib)
accumulate
SC_CLK_TCKc                   / p\         P                  ! R4       F  pVP                  4       '       g   K   \        RV R2R4      ;_uu_ 4       pVP	                  4       P                  R^4      ^,          P                  4       pRRR4       VP                  \        X^,          4      . 4      P                  \        V4      4       K  	  . V .reV'       dE   VP                  4       pVP                  V4       VP                  VP                  V. 4      4       KL  V#   + '       g   i     L; i  \        \        \        3 d     EK/  i ; i)z-Every live descendant of root_pid, inclusive.z/proc/proc//statrb   )N)oslistdirisdigitopenreadrsplitsplit
setdefaultintappendOSError
IndexError
ValueErrorpopextendget)root_pidchildrenentryhandlefieldsseenstackpids   &       V/home/user/overdeck-private-tests/workstation-ai-broad/modules/monitor/bin/stall-guardread_proc_treer#   %   s    HG$}}	ugU+T22f--dA6q9??A 3F1I3::3u:F % xj%
iikCX\\#r*+K 32 Z0 		s*   D.6DAD.D+&D..E	E	c                   ^ ;r\        V 4       F  p \        RV R2R4      ;_uu_ 4       pVP                  4       P                  R^4      ^,          P	                  4       pRRR4       V\        X^,          4      \        V^,          4      ,           ,          p \        RV R2R4      ;_uu_ 4       pV F2  pVP                  R4      w  rxp	VR9   g   K   V\        V	4      ,          pK4  	  RRR4       K  	  W3#   + '       g   i     L; i  \        \        \        3 d     Li ; i  + '       g   i     EK,  ; i  \        \        3 d     EKD  i ; i)	z3Summed cpu jiffies and syscall bytes over the tree.r   r   r   r	   Nz/io   :)s   rchars   wchar)
r#   r   r   r   r   r   r   r   r   	partition)
r   cpuio_bytesr!   r   r   linename_values
   &         r"   sample_treer-   9   s)   Ch'	se5)400F--dA6q9??A 13vbz?S_44C	se3'..&"D%)^^D%9NDU33 CJ. # /. ( = 10 Z0 		 /..
 $ 		s^   D6C:!8DD>2D)D)+D>:D
DD&%D&)D;4D>;D>>EEc                 f     \         P                  P                  V 4      #   \         d     ^ # i ; i)    )r
   pathgetsizer   r0   s   &r"   size_ofr3   N   s,    wwt$$    ! 00c                 f     \         P                  P                  V 4      #   \         d     R # i ; i        )r
   r0   getmtimer   r2   s   &r"   mtime_ofr9   U   s.    ww%% r4   c                  a a \        S 4      ^8  d   R# S R,          o\        S R4      pVV 3R lpV'       dS   V! V4      pVeG   VR,          V,          VR,          R,          8X  d$   RSR,          S V,          R,          ,
          R R	2# V! V4      pVf   R# \        ;QJ d    R
 S  4       F  '       g   K   RM	  RM! R
 S  4       4      p\        ;QJ d    R S  4       F  '       g   K   RM	  RM! R S  4       4      p. . rRRV3RV3R3 FT  w  rV'       g   K  VP                  V
4       W:,          V,          W:,          R,          8X  g   KC  V	P                  V
4       KV  	  V'       d   \        V	4      \        V4      8w  d   R# SR,          S V,          R,          ,
          pRRP	                  V4       RVR R2# )a  Pure detector. Returns a stall reason, or None.

Two independent rules:
  1. every tracked signal flat across the trailing idle_window
  2. `out` alone flat across output_ceiling_s, whatever cpu/io do

Rule 2 exists because rule 1 ANDs the signals, so a spin loop, retry loop
or keepalive that moves ONE signal forever would veto detection for good.
An unreadable counter is not treated as flat-and-therefore-stalled: `cpu`
or `io` all-zero for the whole run disables that signal rather than firing.
Noutr'   ioc                z   < SR,          V ,
          pRp\        S4       F  w  r4VR,          V8  d    V# TpK  	  V# )zBNewest sample at or before the window start, or None if too young.tN)	enumerate)window_swindow_startfoundisamplelastsampless   &    r"   anchor_indexclassify.<locals>.anchor_indexm   sI    Cy8+"7+IAc{\) E ,     zno output for r>   .0fzs (ceiling)c              3   2   "   T F  qR ,          x  K  	  R# 5i)r'   N .0ss   & r"   	<genexpr>classify.<locals>.<genexpr>   s     -WU88W   TFc              3   2   "   T F  qR ,          x  K  	  R# 5i)r<   NrL   rM   s   & r"   rP   rQ      s     +7aD''7rR   zno progress on +z for rO   )r;   r'   r<   beat)r;   T)rV   T)lenr   anyr   join)rF   idle_window_soutput_ceiling_s
cumulativerG   indexcpu_everio_evertrackedflatkeyenabledidle_forrE   s   f&&          @r"   classifyre   \   s    7|a2;DG%ABJ -.E!25!9Z=Nr=R!R#DIs0C$CC#HTT'E}s-W-sss-W--Hc+7+ccc+7++GT&(9D'?N[s?5!Z_R%88KK \ c$i3w</Cy75>#..HSXXg./uXcN!DDrI   c                  a  V 3R lpV! \         P                  4       \        P                  ! 4       V,           p\        P                  ! 4       V8  df   \	        S 4       Uu. uF/  p\
        P                  P                  RV 24      '       g   K-  VNK1  	  up'       g   R# \        P                  ! R4       K  V! \         P                  4       R# u upi )zSIGTERM the whole tree, wait for it to actually die, then SIGKILL.

killpg alone misses grandchildren that called setsid, and an unconditional
sleep before SIGKILL lets a recycled PGID absorb the second signal.
c                    < \        S4      p \        P                  ! \        P                  ! S4      V 4       V F  p \        P
                  ! W 4       K  	  R #   \         d     L/i ; i  \         d     K?  i ; i)N)r#   r
   killpggetpgidr   kill)sigpidsr!   r   s   &  r"   
signal_allkill_tree.<locals>.signal_all   sm    h'	IIbjj*C0 C!   		
  s"   +A A,A)(A),A;:A;r   Ng      ?)
signalSIGTERMtime	monotonicr#   r
   r0   existssleepSIGKILL)r   grace_srm   deadlineps   f&   r"   	kill_treery      s    
 v~~~~')H
..
X
%)(3T3arww~~qcl7S3TT

3v~~ Us   !*CCc                 *   V P                  R V R24       V P                  RV R24       V P                  R4       VR%R  FL  pV P                  RVR,          R RVR	,          R
 RVR,          R RVR,          R RVR,          R R24       KN  	   \        V4      pV P                  RV R24       VR,           F  p \        RV R2R4      P                  4       P	                  RR4      P                  RR7      p\        RV R2R4      P                  4       P                  R^4      ^,          P                  4       ^ ,          P                  4       p	\        RV R24      P                  4       P                  4       ;'       g    Rp
V P                  RV RV	 R V
 R!VR",           R2	4       K  	   \        VR4      ;_uu_ 4       pVP                  \        ^ \        V4      R#,
          4      4       VP                  4       P                  RR7      pRRR4       V P                  R$X,           R,           4       V P                  4        R#   \         d     EK  i ; i  \         d     Li ; i  + '       g   i     Le; i  \         d     LVi ; i)&z
[stall-guard] STALL DETECTED: 
z[stall-guard] log: z8[stall-guard] trailing samples (t, out, cpu, io, beat):
Nz  r>   z8.0f r;   12dr'   10dr<   14drV   rJ   z[stall-guard] live pids: :N   Nr   z/cmdliner           replace)errorsr   r	   z/wchan-z  pid z state=z wchan=z :: :Nx   Ni  z[stall-guard] last output:
i)writer#   r   r   r   decoder   r   stripr   seekmaxr3   flush)streamrF   log_pathreasonr   rO   treer!   cmdstatewchanr   tails   &&&&&        r"   dump_diagnosticsr      sf   
LL3F82>?
LL&xj34
LLLMRS\r!C&a%~QqxnAagc]RSTUV\T]^aSbbdef h'0b9:99CVC5148==?GGtT[[cl[mvcU%0$7<<>EEdANqQWWYZ[\ccevcU&12779??AHHSvcU'%wd3t9+UWXY (D!!VKKAwx04789;;=''y'9D " 	3d:TAB LLN    "!  sb   ,I  ?C0I/I  4J AI1'J II  II   I.-I.1J	<J JJc                    \         P                  P                  R 4      R8w  d)   \        P                  P                  RV  RV R24       R#  \        P                  ! RRRW.^
R	7       R#   \        \        P                  3 d     R# i ; i)
STALL_GUARD_NOTIFY1z/[stall-guard] desktop notification suppressed: u    — r{   Nznotify-sendz-ucriticaltimeout)
r
   environr   sysstderrr   
subprocessrunr   SubprocessError)titlebodys   &&r"   notifyr      sz    	zz~~*+s2

J5'QVW[V\\^_`tZErRZ//0 s   A, ,B
Bc           	      N  a aaaa	a
aaa S P                   ;'       gl    \        P                  P                  \        P                  P                  R 4      ;'       g    RRS P                   R\        P                  ! 4        R24      o\        P                  ! \        P                  P                  S4      RR7       S P                  ;'       gj    \        P                  P                  \        P                  P                  R4      S P                   R\        \        P                  ! 4       4       R24      o\        P                  ! \        P                  P                  S4      RR7       \        SR	4      p\        P                   ! R
RS.\        P"                  R7      p\        P                   ! S P$                  VP&                  \        P(                  RR7      oVP+                  4        . o	RRRR/o. o\        P,                  ! 4       o
\.        P0                  ! 4       oV VVVV	V
VVV3	R lp\.        P2                  ! VRR7      pVP5                  4        SP7                  4       pSP9                  4        VP                  S P:                  ^,           R7        VP&                  P+                  4        VP7                  ^
R7       \@        PB                  PE                  RS R\G        S	4       R24       SR,          '       dC   S PH                  '       d1   \@        PB                  PE                  RSR,           R24       \J        # SR,          '       d+   \@        PB                  PE                  RSR,           R24       V#   \<        \        P>                  3 d     Li ; i)XDG_RUNTIME_DIRz/tmpzstall-guard-r   z.logT)exist_okz~/.cache/stall-guardz.jsonlwbteez-a)stdin)stdoutr   start_new_sessionstalledFr   Nc                    <	 SP                  SP                  4      '       Eg   \        P                  ! 4       S,
          p \	        SP
                  4      w  rR T R\        S	4      RTRTRSP                  '       d   \        SP                  4      MR/pSP                  V4       \        SR4      ;_uu_ 4       pVP                  \        P                  ! V4      R,           4       RRR4       SP                  '       d   V SP                  8  d   S
'       g{   S
P                  V 4       R	V ^<,          R
 R2p\        P                   P                  RV R24       \        P                   P#                  4        \%        RSP&                   R2V4       \)        SSP*                  SP,                  4      pVf8   SP.                  '       d&   V SP.                  8  d   RV R
 RSP.                   R2pV'       g   EK  SR,          '       d   EK	  RSR&   VSR&   \1        \        P                   SS	VSP
                  4       \%        RSP&                   R2V4       SP2                  '       d#   \5        SP
                  SP6                  4       R# \        P                   P                  R4       EK  R#   + '       g   i     EL; i)r>   r;   r'   r<   rV   r7   ar{   Nzstill running after rJ   um   m — slower than 99.9% of past runs. Progress signals are moving, so this is NOT a stall; check it yourself.z[stall-guard] SLOW: zstall-guard: z unusually slowzwall-clock deadline exceeded (zs > zs)r   Tr   z stuckz8[stall-guard] observe mode: NOT killing; recording only
)waitsample_intervalrq   rr   r-   r!   r3   	heartbeatr9   r   r   r   jsondumpswall_noticer   r   r   r   rb   re   idle_windowoutput_ceilingwall_deadliner   enforcery   
kill_grace)nowr'   r(   rD   r   noticer   argschildr   noticedrF   startedstop
trace_pathverdicts          r"   watchcmd_run.<locals>.watch   s#   ))D0011.."W,C'		2MCSwx(shDNNN0F NN6"j#&&&TZZ/$67 ' C$*:*:$:7s#*38C. 9^ _  

  #7xr!BC

  "txxj@&Igt'7'79L9LMF~$"4"4"4t?Q?Q9Q9#c$tGYGYFZZ\]vgi00%)	"$*! Wh		Rtxxj7@<<<eii9

  !\]G 2 '&&s   8-J..J?	)targetdaemonr   z[stall-guard] trace  (z
 samples)
z[stall-guard] FAILED LOUDLY: r{   z![stall-guard] would have failed: z (observe mode)
)&logr
   r0   rY   r   r   rb   getpidmakedirsdirnametrace
expanduserr   rq   r   r   PopenPIPEcommandr   STDOUTcloserr   	threadingEventThreadstartr   setr   r   r   r   r   r   rW   r   
STALL_EXIT)r   r   r   r   watcherstatusr   r   r   rF   r   r   r   r   s   f     @@@@@@@@r"   cmd_runr      s   xx  277<<


()33V|DHH:Qryy{m[_5`H KK)D9  rww||
12txxj#diikBRASSY4Z J KK
+d;
x
C


E42*//
JCT\\#))JDUDUimnEIIKG%40GGnnG??D$^ $^L eD9GMMOZZ\FHHJLL--1L2		 JJ+J<r#g,{STydlll

89J8K2NOy

<WX=N<OO`abM Z//0 s   ,N N$#N$c                   . pV P                    EF)  p. p\        V4      ;_uu_ 4       pV FB  pVP                  4       pV'       g   K  VP                  \        P
                  ! V4      4       KD  	  RRR4       Rp\        ^\        V4      ^,           4       F1  p\        VRV V P                  V P                  4      pV'       g   K1   M	  V'       d   RMRpV'       d   VP                  V4       \        VR R\        P                  P                  V4       R\        V4       RT;'       g    R 24       EK,  	  \        R	\        V4       R
\        V P                   4       RV P                   R24       V P                  R8X  d   V'       d   \        R4       ^# V P                  R8X  d1   \        V4      \        V P                   4      8w  d   \        R4       ^# ^ #   + '       g   i     ELy; i)zOffline detector verification over recorded traces.

Both arms are required before arming enforcement: zero firings across traces
of runs that completed successfully, and confirmed firing on known-stuck ones.
NFIREDquiet6r|   r   z
 samples)  r{   z of z traces fired at idle_window=rO   u9   FALSE POSITIVES — do not arm enforcement at this windowfiredu:   MISSED DETECTIONS — detector does not catch known stalls)tracesr   r   r   r   loadsrangerW   re   r   r   printr
   r0   basenameexpect)	r   r   r0   rF   r   r)   r   rC   r   s	   &        r"   
cmd_replayr   !  s    E$ZZ6zz|4NN4::d#34  
 q#g,*+Agbqk4+;+;T=P=PQFv , #LL
!BGG,,T232c'l^:fllXZ^\]   
Bs5zl$s4;;/00MdN^N^M__`
ab{{g%IJ{{g#e*DKK0@"@JK+ ZZs   G!)G!!G2c                  ,   \         P                  ! \        \         P                  R 7      p V P	                  RRR7      pVP                  RRR7      pVP                  RRRR	7       VP                  R
\        RRR7       VP                  R\        RRR7       VP                  R\        RRR7       VP                  R\        RRR7       VP                  R\        RR7       VP                  RRR7       VP                  RRR7       VP                  RRR7       VP                  R\        RR7       VP                  R R!R"R#7       VP                  R$\         P                  R%7       VP                  \        R&7       VP                  R'R(R7      pVP                  R)R*R%7       VP                  R
\        RR7       VP                  R\        RR7       VP                  R+. R1OR,R-7       VP                  \        R&7       V P                  4       pVP                  R8X  df   VP                  '       d1   VP                  ^ ,          R.8X  d   VP                  R/,          Vn        VP                  '       g   V P                  R04       VP!                  V4      # )2)descriptionformatter_class
subcommandT)destrequiredr   zsupervise a command)helpz--keyz(short label for traces and notifications)r   r   z--idle-windowg      @z4seconds of total flatness before stall (default 900))typedefaultr   z--output-ceilingr7   zBseconds of zero output before stall, whatever cpu/io do (0 = none)z--wall-noticez=seconds after which to warn LOUDLY without killing (0 = none)z--wall-deadlinez)hard wall-clock cap in seconds (0 = none)z--sample-intervalg      .@)r   r   z--heartbeatz;file a legitimate long wait touches to declare itself alivez--logzcaptured output pathz--tracezsample trace path (JSONL)z--kill-graceg      $@z	--enforce
store_truez&kill on stall; default is observe-only)actionr   r   )nargs)funcreplayz(re-run the detector over recorded tracesr   rT   z--expectrX   )choicesr   z--:   NNzrun requires a command after --)r   r   rX   )argparseArgumentParser__doc__RawDescriptionHelpFormatteradd_subparsers
add_parseradd_argumentfloat	REMAINDERset_defaultsr   r   
parse_argsr   r   errorr   )parsersubr   r   r   s        r"   mainr  B  sM   $$(JnJnoF


\D

AC
..%:.
;CWt2\]_5%F|}'eS  HL  M_5#  ED  E&UCFqr(udC])fgW#9:Y%@A^%>[<deYh&8&89'"^^H+U^VF
,
eUC
*D

,EuU
Z(D%<<<DLLOt3<<+DL|||LL:;99T?rI   __main__r6   )!r   r   r   r
   ro   r   r   r   rq   r0   insertrY   r   realpath__file__stall_detectr   r   sysconfCLOCK_TICKSr#   r-   r3   r9   re   ry   r   r   r   r   r  __name__exitrL   rI   r"   <module>r     s   &   	   
   277<<0@0@0J KTSXY Z #
jj&(*1Eh6:L^B F zHHTV rI   