
    N jN                       S SK Jr  S SKrS SKrS SKrS SKrS SKJr  S SKJ	r	J
r
  S SKJrJr  S SKrS SKJr  S SKJr  S SKJr  S SKJr  S	S
KJrJr  \(       a  S SKJrJr  \" 5       r\ " S S5      5       r " S S\R                   R@                  5      r!SS jr"        SS jr#      SS jr$SS jr%SS jr&\ " S S5      5       r'\%4SSSSSS.                   SS jjjr(g)     )annotationsN)	dataclass)partialwraps)AnyTYPE_CHECKING)tqdm)StorageWeakRef)ContentStoreWriter   )get_outputsget_placeholders)CallableSequencec                  >    \ rS rSr% S\S'   S\S'   S\S'   S\S'   S	rg
)LoadTensorMeta   ztuple[int, ...]sizestrideztorch.dtypedtypeztorch.devicedevice N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r       m/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/torch/_functorch/fx_minifier.pyr   r      s    
r   r   c                  d   ^  \ rS rSrSSS.       S	U 4S jjjrS
U 4S jjrSU 4S jjrSrU =r$ )ConcreteProp#   NFwriterskip_offloadc               h   > [         TU ]  U5        X l        X0l        [	        5       U l        S U l        g N)super__init__r%   r&   setseen_storagespbar)selfmodr%   r&   	__class__s       r    r*   ConcreteProp.__init__$   s.     	(25%	r   c                  > U R                   R                  S5        [        TU ]  U5      nUR                  n[        U[        R                  5      (       Ga   U R                  c  X!R                  S'   U$ [        UR                  5       5      U R                  ;   a  S UR                  S'   U$ U R                  (       d:  U R                  R                  [        R                   R#                  SU5      U5        [%        UR'                  5       UR)                  5       UR*                  UR,                  5      UR                  S'   U R                  R/                  [        UR                  5       5      5         U$ [0        UR                  S'   U$ )Nr   concrete_valueeager)r-   updater)   run_nodename
isinstancetorchTensorr%   metar
   untyped_storager,   r&   write_tensorospathjoinr   r   r   r   r   addis_tuple)r.   nrr7   r0   s       r    r6   ConcreteProp.run_node1   s2   		GQvva&&{{"+,'(   "!"3"3"56$:L:LL 04AFF+,   ,,00gt1LaP/=!((*aggqxx0AFF+, &&**>!:K:K:M+NO  (0AFF#$r   c                  > U R                   n[        U[        R                  5      (       d  [	        S[        U5       35      e[        S[        UR                  R                  5      U R                  S L S9 nX0l        [        TU ]4  " U6 nU R                  (       d  UR                  S5        UsS S S 5        $ ! , (       d  f       g = f)Nzexpected fx.GraphModule, got z(Saving intermediates for delta debugging)desctotaldisablezESaved!  To skip next time, run with --skip-saving-eager-intermediates)moduler8   fxGraphModuleAssertionErrortyper	   lengraphnodesr%   r-   r)   runr&   set_description)r.   argsr/   r-   rD   r0   s        r    	propagateConcreteProp.propagateJ   s    kk#r~~.. #@c!LMM;ciioo&KK4'
 IT"A$$$$[ 
 
 
s   87B99
C)r-   r,   r&   r%   )r/   fx.GraphModuler%   zContentStoreWriter | Noner&   boolreturnNone)rC   fx.NoderY   r   )rT   r   rY   r   )	r   r   r   r   r*   r6   rU   r   __classcell__)r0   s   @r    r"   r"   #   sJ    
 -1" *	
  
 2 r   r"   c                    U R                   S:H  =(       a:    U R                  [        R                  R                  R
                  R                  L $ )Ncall_function)optargetr9   ops
debugprimsload_tensordefault)nodes    r    is_load_tensor_noderf   \   s;    ?" 	DKK599//;;CCCr   c                :   UR                   S:X  d  UR                   S:X  a  g[        U5      (       a  gUR                  R                  SS 5      n[	        U[
        R                  5      (       a8  SUl         UR                  Ul        SUl	        0 Ul
        UR                  U5        gUc  gU[        L a5  Sn[        UR                  5       H  n[        XU5      =(       d    UnM     U$ [	        U[         5      (       a  SUl         [
        R"                  R$                  R&                  R(                  Ul        [*        R,                  R/                  SUR                  5      UR0                  UR2                  4Ul	        UR4                  UR6                  S	.Ul
        gg)
NoutputplaceholderFr3   r   Tr^   r4   )r   r   )r_   rf   r;   getr8   r9   r:   r7   r`   rT   kwargsappendrB   listusers_convert_node_to_placeholderr   ra   rb   rc   rd   r>   r?   r@   r   r   r   r   )rP   re   inpsconcrete_valrD   
tuple_users         r    ro   ro   d   sI    ww(dgg64  99==!148L,--ii	L!				!tzz*J,UEJA + 	L.	1	1!ii**66>>GGLL$)),
	 #))!''
 r   c                |    [         R                  " 5        S3n[         R                  " USS9  SSKJn  U" XU5        g)z
Takes minified FX graph as primary input, and ports it to HLO via StableHLO
Provides minified HLO graph as output, and archive them to local directory
z
/hlo_filesT)exist_okr   )save_torch_model_as_stablehloN)r>   getcwdmakedirstorch_xla.stablehloru   )minified_fx_graphinputshlo_dirru   s       r    create_minified_hlo_graphr|      s2     Z(GKK$'A!"3WEr   c                    [        S[        U R                  R                  5       SU Vs/ s H/  o"R                  UR
                  UR                  R                  4PM1     sn SU R                   S35        g s  snf )Nz
# Working Repro with z nodes
inps = zo
inps = [torch.zeros(())] + [torch.ones(shape, dtype=dtype, device=device) for (shape, dtype, device) in inps]

)	printrO   rP   rQ   shaper   r   rN   code)fx_grp   is      r    
dump_stater      sp    	$****+, -489Dq''177AHHMM	*D9: ; 		 	:s   6A9c                $    U S:X  a  gX S-
  -  S:H  $ )Nr   Fr   r   )rC   s    r    is_power_of_twor      s    AvQKAr   c                  4    \ rS rSr% S\S'   S\S'   S	S jrSrg)

ReproState   fx.GraphrP   Sequence[torch.Tensor]rp   c                    [        U R                  5      n[        U5      [        U R                  5      :w  a-  [	        S[        U5       S[        U R                  5       35      eg )Nlen(ph_nodes)=z != len(self.inps)=)r   rP   rO   rp   rM   )r.   ph_nodess     r    __post_init__ReproState.__post_init__   sT    #DJJ/x=C		N*  X/B3tyy>BRS  +r   r   N)rY   rZ   )r   r   r   r   r   r   r   r   r   r    r   r      s    O
  r   r   F)save_diroffload_to_diskr&   skip_sanitymax_granularityc          
       ^ ^^^^^^^^^^^^^^  T R                   n	[        U	R                  5      n
Ub  [        U5      (       d  [	        SU S35      eSmS)U 4S jjmS*U UU4S jjmSnU(       a  Uc  [        S5      e[        U5      n[        T XS9R                  " U6   U(       d  T" X5      (       d  [	        S	5      e[        S
U
 S3[        R                  S9        S+UU4S jjm    S,U4S jjnU" S5              S-U4S jj5       mU" S5              S-U4S jj5       mS.S jm S.UU 4S jjm        S-U4S jjnU" S5      " U5      mU" S5              S-U4S jj5       m      S/S jmU" S5              S-UUUU 4S jj5       mU" S5              S-UU4S jj5       m[        X5      n        S0UUUUUU4S jjn U" [        R                  " T UR                   5      UR                  5        [!        S ["        R$                  " ["        R&                  " [        UR                   R                  5      5      5      -  5      nUb  [)        UU5      nU" UUSS!9nUb  UnM  US -  nS"nUS#:  a  U" UUS"S!9nUb  UnSnOUS -  nUS#:  a  M  U(       a  M  T" US#5      nUb  UnM   T" UR                   UR                  5      (       d  [	        S$5      e[        S%T S&3[        R                  S9  [        R                  " T UR                   5      nS'[*        R,                  ;   a  [/        UUR                  5        U" UUR                  5        [        S([        R                  S9  UUR                  4$ )1a  
Minimizes a FX graph with given inputs, such that the resulting FX graph still returns True for module_fails.

Does 2 main strategies:
1. Truncates suffix: Removes some suffix from the graph and sets a new output.
2. Delta Debugging: Tries replacing half of the graph with inputs. If fails,
    tries replacing quarter of the graph, etc.

>>> # xdoctest: +SKIP(failing)
>>> failing_function = fx.symbolic_trace(f)
>>> minimize(failing_function, [torch.randn(5)], lambda fx_g, inps: fx_g(*inps))

note: module_fails returns True if it fails.
Nzmax_granularity z not power of twor   c                n   > [         R                  " T[        R                  " U 5      5      R                  $ r(   )rK   rL   copydeepcopyrP   )fx_graphfail_fs    r    deepcopy_fx_graph#minifier.<locals>.deepcopy_fx_graph   s#    ~~fdmmH&=>DDDr   c                   > [         R                  " U 5      n TS-  m[        R                  " TU 5      nUR                  R                  5         T" X!5      $ )Nr   )r   r   rK   rL   rP   lint)rP   rp   r/   r   module_failsnum_queriess      r    graph_failsminifier.<locals>.graph_fails   sD    e$qnnVU+		C&&r   z3save_dir must not be None when offload_to_disk=Truer$   z#Input graph did not fail the testerzStarted off with  nodesfilec                B   >^ ^ [        T 5      SSUUUU 4S jjj5       nU$ )Nc                D  > [        [        R                  S9  [        ST SU S[        U R                  R
                  5       S[        U R                  5       S3	[        R                  S9  T" T
" U R                  5      [        U R                  5      U5      nUGbg  [        UR                  R
                  5      n[        U R                  R
                  5      n[        UR                  5      n[        U R                  5      n[        [        UR                  5      5      n[        [        U R                  5      5      nSn	X4:  a!  Sn	[        S	U S
U S3[        R                  S9  XV:  a!  Sn	[        S	U S
U S3[        R                  S9  Xx:  a!  Sn	[        S	U S
U S3[        R                  S9  U	(       d  [        S5      eT" UR                  UR                  5      (       d  [        S[        R                  S9  g U$ [        ST 3[        R                  S9  g )Nr   z
Strategy: z (G: z) (z nodes, z inputs)FTzSUCCESS: Went from z to r   z inputsz outputsz$Success raised but no progress made?z=WARNING: Something went wrong, not applying this minificationzFAIL: )
r   sysstderrrO   rP   rQ   rp   rm   r   RuntimeError)	old_stategranularity	new_state	new_nodes	old_nodesnew_inpsold_inpsnew_outsold_outsprogress_mader   r   r7   strategys             r    new_func6minifier.<locals>._register_strategy.<locals>.new_func   s   szz"TF%} 5	--./xINN8K7LHVZZ
 !!)//2D4H+I $	 5 56		 5 56	y~~.y~~.{9??;<{9??;< %($(M-i[YKvN ZZ &$(M-hZtH:WM ZZ &$(M-hZtH:XN ZZ
 %&'MNN"9??INNCCW ZZ    tfoCJJ7r   )r   )r   r   r   intrY   ReproState | None)r   )r   r7   r   r   r   s   `` r    _register_strategy$minifier.<locals>._register_strategy   s(     
x1	 1	 
1	f r   c                   > [        TU S9$ )N)r7   )r   )r7   r   s    r    register_strategy#minifier.<locals>.register_strategy6  s     )55r   zTruncate suffixc                   >^	 [        5       n[        R                  " 5       n0 m	[        U R                  5       H  u  pVUR                  UU	4S j5      nUR                  S;  a  XR-  S:X  a  XRS-  -  S:w  a  XS;  az  UR                  U45      n[        UR                  5      [        U R                  5      :  a  T
" XA5      (       a  [        XA5      s  $ UR                  U5        UR                  U5        UT	U'   M     g )Nc                   > TU    $ r(   r   xenvs    r    <lambda>1minifier.<locals>.remove_suffix.<locals>.<lambda>F  s	    3q6r   )ri   rh   r      )r+   rK   Graph	enumeraterQ   	node_copyr_   rh   rO   r   rA   
erase_node)	cur_graphcur_inpsr   tested	new_graphidxre   new_nodeoutput_noder   r   s            @r    remove_suffixminifier.<locals>.remove_suffix>  s     5HHJ	&("9??3IC **41ABHww77 %*a0A5)"+"2"2H;"?K9??+c)//.BB{!H H  *)>>

3!,,[9 CI# 4$ r   zRemove outputsc                F  > [        SUS-  5      nS n[        U R                  5       H  u  pEXEl        UR                  S:X  d  M  Un  O   Uc  g [        UR                  S   [        R                  5      (       a  g UR                  S   n[        U[        [        45      (       d  [        S[        U5       35      e[        US S9n[        U5      S:X  a  g [        S[        U5      U5       H0  nUS U XtU-   S  -   4Ul        T" X5      (       d  M%  [!        X5      s  $    g )Nr   r   rh   r   z2expected output_args_raw to be list or tuple, got c                n    [        U [        R                  5      (       a  U R                  $ [	        S5      $ )Ng    eA)r8   rK   Noder   r   )r   s    r    r   2minifier.<locals>.remove_outputs.<locals>.<lambda>s  s$    :a#9#9!%%Gs3xGr   )key)maxr   rQ   r   r_   r8   rT   rK   r   rm   tuplerM   rN   sortedrO   ranger   )	r   r   r   rh   r   re   output_args_rawoutput_argsr   s	           r    remove_outputs minifier.<locals>.remove_outputsY  s$    ![A-.!%"9??3ICHww("	 4 >fkk!nbgg.. !++a./D%=99 DT/EZD[\  G
 {q C,k:C&t,{;L;N/OOQFK9//!)66 ; r   c                   U R                   nU R                  n[        [        U5      5      n[	        U5      [	        U5      :w  a#  [        S[	        U5       S[	        U5       35      e/ n[        [	        U5      5       HF  n[	        X5   R                  5      S:X  a  UR                  X5   5        M3  UR                  X%   5        MH     [	        U5      [	        U5      :  a  [        X5      $ g )Nr   z != len(cur_inps)=r   )rP   rp   rm   r   rO   rM   r   rn   r   rl   r   )	cur_stater   r   r   r   r   s         r    remove_unused_inputs_unchecked0minifier.<locals>.remove_unused_inputs_unchecked~  s    OO	>>(34x=CM)  X/A#h-Q  (*X'C8=&&'1,$$X]3.	 (
 x=3x=(i22r   c                d   > T" U 5      nUb$  T" UR                   UR                  5      (       a  U$ g r(   )rP   rp   )r   r   r   r   s     r    remove_unused_inputs_checked.minifier.<locals>.remove_unused_inputs_checked  s0    29=	 [)..%Q%Qr   c                &   > T" [        X5      5      $ r(   )r   )r   r   r   r   s      r    _remove_unused_wrapper(minifier.<locals>._remove_unused_wrapper  s     ,Jy,KLLr   zRemove unused inputszEliminate dead codec                `   > U R                  5       (       a  T" X5      (       a  [        X5      $ g r(   )eliminate_dead_coder   )r   r   r   r   s      r    r   %minifier.<locals>.eliminate_dead_code  s,     ((**{9/O/Oi22r   c                J  ^ [         R                  " 5       n0 mSnU R                   H  nUR                  S:X  a  UR	                  UU4S j5      nUTU'   M0  U(       d  [        U5      (       ay  UR                  UR                  5      nUTU'   UR                  [        R                  R                  R                  R                  " UR                  0 UR                  D65        M  SnM     U R                   H&  nUT;  d  M  UR	                  UU4S j5      nUTU'   M(     U$ )NFri   c                   > TU    $ r(   r   r   s    r    r   =minifier.<locals>._consolidate_placeholders.<locals>.<lambda>  	    s1vr   Tc                   > TU    $ r(   r   r   s    r    r   r     r   r   )rK   r   rQ   r_   r   rf   ri   r7   rl   r9   ra   rb   rc   rd   rT   rk   )r   rp   r   seen_non_placeholderre   r   r   s         @r    _consolidate_placeholders+minifier.<locals>._consolidate_placeholders  s     HHJ	$
 OODww-'$..t5EF$D	).A$.G.G$00;$D	II((44<<diiW4;;W (,$ $ OOD3$..t5EF$D	 $ r   zDelta Debuggingc                (  > [        U R                  5      n[        SX25       H  nSnT" U 5      n[        US S  5      n[	        X4U-   5      n[        XH5       H0  n	[        UR                  5      U	   n
[        XjU5      (       d  M.  SnM2     U(       d  Mq  UR                  5         T" Xg5      nT" [        Xg5      5      nUc  [        Xg5      nT" UR                  UR                  5      (       d  M  [        UR                  UR                  5      s  $    g )Nr   FT)
rO   rQ   r   rm   minro   r   r   rP   rp   )r   r   r   	num_nodesstart_rangeis_removingr   r   	end_ranger   r   r   r   r   r   r   s               r    delta_debugging!minifier.<locals>.delta_debugging  s     	(	 I;KK))4IHQK(HI['@AI[4	05/	XNN"&K 5 ))+1)FI6z)7VWI &y;	9??INN;;!)//9>>BB# <& r   zConsolidate Inputsc                   > [        U5      n[        US S  5      nT" X5      n [        U5      U:  a  T" X5      (       a  [        X5      $ g r(   )rO   rm   r   )r   r   r   old_lenr   r   r   s        r    consolidate_inputs$minifier.<locals>.consolidate_inputs  sJ     h-$-iB	x=7"{9'G'Gi22r   c                2  > [        SU 3[        R                  S9  / n[        U R                  R
                  5      n[        [        U R                  5      5      nXTS-  :  a  UT/-  nU(       a  UT
TT/-  nUTT	/-  nU H  nU" X5      nUc  M  Us  $    g )NzTrying granularity r   r   )r   r   r   rO   rP   rQ   r   )failing_stater   use_non_granular
strategiesr   num_outputsr   r   r   r   r   r   r   remove_unused_inputss           r    try_granularity!minifier.<locals>.try_granularity  s     	#K=1

C
++112	+m&9&9:;a'>**J#$" J 	}o66
"H <I$   # r   Tr   )r  Fr   z9Uh oh, something went wrong :( Final graph is not failingzMade z queriesXLA_HLO_DEBUGz#Wrote minimal repro out to repro.py)r   r   rY   r   )rP   r   rp   r   rY   rX   )r   zDCallable[[fx.Graph, Sequence[torch.Tensor], int], ReproState | None]r7   strrY   z.Callable[[ReproState, int], ReproState | None])r7   r  rY   zCallable[[Callable[[fx.Graph, Sequence[torch.Tensor], int], ReproState | None]], Callable[[ReproState, int], ReproState | None]])r   r   r   r   r   r   rY   r   )r   r   rY   r   )r   r   rp   list[torch.Tensor]rY   r   )r  r   r   r   r  rX   rY   r   )rP   rO   rQ   r   r   rM   r   r"   rU   r   r   r   r   rK   rL   rp   r   mathfloorlog2r   r>   environr|   )!r   rp   r   r   r   r   r&   r   r   failing_graphcur_sizer%   r   r   r  r  r   r   has_progress
failing_fxr   r   r   r   r   r   r   r   r   r   r  r   r   s!   ` `                 @@@@@@@@@@@@@r    minifierr     sj   6 LLM=&&'H"??+K+K-o->>OPQQKE' ' F !VWW#H-BLLdS{=??@AA	hZv
.SZZ@8V88 
88 8t66
6 ()'=LO	 *4 '(""'="LO"	" )"H& MM'=MLOM	M
 --CD ,-'=LO	 .#5	< ()'=LO	  *2 +,'=LO	 - }3M!03GK	 4 2>>&-*=*=>@R@RS!

499S9L9L9R9R5S+T UVW&o{;K#M;QUV	 %MQ'{UI $ )#AK Q "=!4	 %M}**M,>,>??VWW	E+h
'cjj9(;(;<J "**$!*m.@.@Az=--.	
/cjjA}))))r   )re   r[   rY   rX   )rP   r   re   r[   rp   r  rY   rX   )ry   rW   rz   r   rY   rZ   )r   rW   rp   r   rY   rZ   )rC   r   rY   rX   )r   rW   rp   r   r   z8Callable[[fx.GraphModule, Sequence[torch.Tensor]], bool]r   z8Callable[[fx.GraphModule, Sequence[torch.Tensor]], None]r   z
str | Noner   rX   r&   rX   r   rX   r   z
int | NonerY   z-tuple[fx.GraphModule, Sequence[torch.Tensor]]))
__future__r   r   r  r>   r   dataclassesr   	functoolsr   r   typingr   r   r9   torch.fxrK   	torch.hubr	    torch.multiprocessing.reductionsr
   torch.utils._content_storer   compile_utilsr   r   collections.abcr   r   objectrB   r   Interpreterr"   rf   ro   r|   r   r   r   r  r   r   r    <module>r"     so   "   	 
 ! $ %    ; 9 8 2 8   6588'' 6r//"/*</	/dF%F/EF	F 	 	 	  LV	o*  !"&o*o*
 o* Ko* I	o* o* o* o* o*  o* 3o*r   