
    N jq                        S SK r S SKrS SKJr  S SKJr  S SKJrJr  S SK	r	S SK
Jr  S SKJr  S SKJr  S SKJr  S	S
/r\R&                  " \5      =rr\" SS9 " S S	5      5       rS\	R0                  R2                  S\S\4S jr\" SS9    SSS.S\S\	R0                  R2                  S\\/\4   S\\\\4      S\\   S\\   S\S\\   4S jjj5       rg)    N)OrderedDict)Callable)AnyOptional)compatibility)lazy_format_graph_code)GraphModule)Node	Partitionsplit_moduleT)is_backward_compatiblec                   0    \ rS rSrS\4S jrS\4S jrSrg)r      namec                     Xl         SU 3U l        / U l        0 U l        0 U l        0 U l        0 U l        [        R                  R                  R                  5       U l	        0 U l        0 U l        g )Nsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr   s     m/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/torch/fx/passes/split_module.py__init__Partition.__init__   sa    	$TF+%'')(*-/+-+088>>+?+?+A
-/')    returnc                     SU R                    SU R                   SU R                   SU R                   SU R                   SU R
                   3$ )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r   s    r    __repr__Partition.__repr__!   sb    TYYK  ' (} % '((,(9(9': ;&&*oo%68	
r#   )
r   r   r   r   r   r   r   r   r   r   N)__name__
__module____qualname____firstlineno__strr!   r&   __static_attributes__ r#   r    r   r      s    
*S 
*
# 
r#   modqualnamer$   c                     U nUR                  S5       H-  n[        X#5      (       d  [        SU S35      e[        X#5      nM/     U$ )N.zNode target z not found!)splithasattrAttributeErrorgetattr)r/   r0   attr_valatoms       r    _get_attr_from_qualnamer9   ,   sJ    Hs#x&& <z!EFF8* $ Or#   )partition_affixmroot_msplit_callbackqualname_mapkeep_original_orderkeep_original_node_namekeep_original_input_namer:   c                !  ^ ^^^^^2^?^@^A^B^C^D^E^F^G^H^I [         R                  S[        ST SS95        S[        S[        [
        [        4   S[        [
        [        R                  R                  R                  4   4U@UU 4S jjnS	S
K
n	0 mG0 mE0 mIS[        S[        [           4UGUI4S jjmHUUGU4S jn
[        R                  R                  [        R                  R                  [        R                  R                   /n[#        5       n[#        5       n0 nS
n[%        5       nT R&                  R(                   GH  m2T2R*                  R-                  S5      =nb  [/        U[        R0                  [        R2                  45      (       aP  [/        UR4                  R6                  =nU	R8                  5      (       a  UTI;  a  T2TIUR4                  R6                  '   T2R:                  S;   a  M  U
" T25        T2R:                  S:X  Ga  T2R<                  U;   Ga  T2R<                  [        R                  R                   L a  [?        T2R@                  5      S:w  a!  [C        S[?        T2R@                  5       35      e[/        T2R@                  S	   [D        5      (       d$  [C        S[G        T2R@                  S	   5       35      eT2n[%        T" T25      15      X'   GO7T2R<                  [        R                  R                  L aX  [I        S T2R@                   5       5      (       d  [C        S5      eURK                  T25        [%        T" T25      15      UT2'   S
UT2'   OT2R<                  [        R                  R                  L a  [?        T2R@                  5      S:w  a!  [C        S[?        T2R@                  5       35      eUT2R@                  S	      RK                  T" T25      5        URM                  T2R@                  S	   5        T2UT2R@                  S	   '   Ub  X   RK                  T" T25      5        U H  nUU   RK                  T" T25      5        M     GM     [I        S URO                  5        5       5      (       d  [C        S5      eURQ                  5        VVs0 s H  u  nnU[S        U5      _M     nnnURQ                  5        VVs0 s H  u  nnU[S        U5      _M     nnn[T        RW                  [X        RZ                  5      (       a,  [T        R                  SU5        [T        R                  SU5        [E        U5      =(       d    [E        U5      nSnT R&                  R(                   GH  m2T2TET2R\                  '   T2R:                  S;   a  M%  T2R:                  S:X  a=  [        R                  R&                  R_                  T2R@                  S	   UH4S j5        Mr  U(       a!  T" T25      nUU:  a  [C        S U S!U 35      eUnT2R<                  U;  d  M  [        R                  R&                  R_                  T2R@                  U2UH4S" j5        [        R                  R&                  R_                  T2R`                  U2UH4S# j5        GM!     [c        TGRe                  5       5      n/ nTGRQ                  5        H3  u  nmF[?        TFRf                  5      (       a  M"  URi                  U5        M5     / nU(       a  URk                  5       nURi                  U5        TGU   Rl                   HH  nTGU   Rf                  Rk                  U5        TGU   Rf                  (       a  M7  URi                  U5        MJ     U(       a  M  [?        U5      [?        TG5      :w  a  [o        S$5      eX4 H  nURQ                  5        H  u  m2n [?        U 5      S	:X  a  [C        S%5      eT2TG[        U S	   5         Rp                  T2'   U SS
  H  n!TG[        U!5         mFTFR&                  Rs                  T2R:                  T2R<                  [u        S& T2R@                   5       5      0 T2RF                  S'9n"T2R*                  Rw                  5       U"l        U"TFRp                  T2'   M     M     M     U GHJ  nTGU   mF0 mDS	mATFRx                   GH%  mCTETC   n#UAUCUUDUEUF4S( jn$U#R:                  S):X  a  [/        U#R<                  [
        5      (       d!  [C        S*[G        U#R<                  5       35      e[{        T U#R<                  5      n%[/        U%[        R|                  R~                  5      (       a?  TFR&                  R                  U#R<                  5      n&U%TFR                  U#R<                  '   OU$" 5       n&OU$" 5       n&TETC   R*                  Rw                  5       U&l        U&TFRp                  TETC   '   GM(     TDTFl<        GMM     T R&                  R(                   GH  m2[        T2S+5      (       d  M  TGT2R                     mFTFRp                  mB[        R                  R&                  R_                  T2R@                  UB4S, j5      n'[        R                  R&                  R_                  T2R`                  UB4S- j5      n(T2R:                  S.;  a  T2R<                  n)Od[{        T T2R<                  5      n*T2R<                  R                  S/S05      n)U*TFR                  U)'   Ub   TFR                   S/U) 3n+T2R<                  UU+'   [/        U'[t        5      (       d  [C        S1[G        U'5       35      e[/        U([        5      (       d  [C        S2[G        U(5       35      eT(       a  T2R\                  OS
n,TFR&                  Rs                  T2R:                  U)U'U(T2RF                  U,S39n"T2R*                  Rw                  5       U"l        U"TFRp                  T2'   GM     U4 H  n[        U5       H  m2UT2   n [?        U 5      S	:X  a  [C        S45      eU S
S  H  n!TG[        U!5         mFUT2   n-U-c  [C        S55      eTFR&                  Rs                  U-R:                  U-R<                  TFRp                  T2   40 U-RF                  S'9n"U-R*                  Rw                  5       U"l        M     M     M     0 n.0 m?[        R                  R&                  R                  5       m@0 n/U(       d+  T R&                  R(                   H  m2U" T2T?U/5      u  m?n/M     O,T R&                  R(                   H  m2T2U.T2R\                  '   M     U(       d  UOUn0[%        5       n1T R&                  R(                   V2s/ s H  n2U2R:                  S6:X  d  M  U2PM     n3n2U0 GHH  nTGU   mF[u        UEUF4S7 jTFR                   5       5      n4[?        U45      n5U5S:X  a  TFR&                  R                  U4S	   5        O=U5S:  a  TFR&                  R                  U45        OTFR&                  R                  S85        U(       a  TFRx                   V6s/ s H  n6U6U3;  d  M  U.U6   PM     n7n6U3 H)  m2T2U1;   a  M  U" T2T?U/5      u  m?n8U1RK                  T25        M+     U7 H)  m2T2U1;   a  M  U" T2T?U/5      u  m?n/U1RK                  T25        M+     [        R                  R                  R                  TFR                  TFR&                  5      U/TFR                  '   T@R                  TFR                  [u        U?4S9 jTFRx                   5       5      5      n9[?        TFR                  5      n:U:S:  a]  [        R                  R                  R                  U95      n;[        TFR                  5       H  u  n<n=U;U<   R4                  T?U='   M     GM  U:S:X  d  GM'  U9T?[        [        TFR                  5      5      '   GMK     U(       a1  T?(       d*  T R&                  R(                   H  m2U" T2T?U/5      u  m?n/M     T R&                  R(                   H_  m2T2R:                  S:X  d  M  T@R                  [        R                  R&                  R_                  T2R@                  S	   U?4S: j5      5        Ma     [        R                  R                  R                  U/T@5      n>[         R                  S[        S;U>SS95        U>$ s  snnf s  snnf s  sn2f s  sn6f )<aM  
Creates subgraphs out of main graph

Args:
    m (GraphModule): Graph module to split
    root_m (torch.nn.Module): root nn module. Not currently used. Included
        because the root nn module is usually transformed via
        torch.fx._symbolic_trace.symbolic_trace (see example below)
    split_callback (Callable[[Node], int]): Callable function
        that maps a given Node instance to a numeric partition identifier.
        split_module will use this function as the policy for which operations
        appear in which partitions in the output Module.
    qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
        mapping from new target names in the module after split to old target
        names in the original module.
    keep_original_order: Optional[bool]: keep the original order of the GraphModule
        or use the Topological order of the new constructed GraphModule
    keep_original_node_name: Optional[bool]: If the partitioned graphs should
        have the same node names as the original graph.
    keep_original_input_name: bool: If the partitioned graphs should
        have the same input names as the original graph.
    partition_affix: Optional[str]: If specified, the submodules' names will contain
        the affix, e.g. "submod_<affix>_<idx>".

Returns:
    GraphModule: the module after split.

Example:

    This is a sample setup:

        import torch
        from torch.fx._symbolic_trace import symbolic_trace
        from torch.fx.graph_module import GraphModule
        from torch.fx.node import Node
        from torch.fx.passes.split_module import split_module

        class MyModule(torch.nn.Module):
            def __init__(self) -> None:
                super().__init__()
                self.param = torch.nn.Parameter(torch.rand(3, 4))
                self.linear = torch.nn.Linear(4, 5)

            def forward(self, x, y):
                z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                w = self.linear(y).clamp(min=0.0, max=1.0)
                return z + w

        # symbolically trace model
        my_module = MyModule()
        my_module_traced = symbolic_trace(my_module)

        # random mod partitioning
        partition_counter = 0
        NPARTITIONS = 3

        def mod_partition(node: Node):
            global partition_counter
            partition = partition_counter % NPARTITIONS
            partition_counter = (partition_counter + 1) % NPARTITIONS
            return partition

        # split module in module with submodules
        module_with_submodules = split_module(
            my_module_traced, my_module, mod_partition
        )

    Output looks like this. Original graph is broken into partitions

        > print(module_with_submodules)
        GraphModule(
            (submod_0): GraphModule(
                (linear): Linear(in_features=4, out_features=5, bias=True)
            )
            (submod_1): GraphModule(
                (linear): Linear(in_features=4, out_features=5, bias=True)
            )
            (submod_2): GraphModule()
        )

        def forward(self, x, y):
            param = self.param
            submod_0 = self.submod_0(x, param, y);  x = param = y = None
            getitem = submod_0[0]
            getitem_1 = submod_0[1];  submod_0 = None
            submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
            getitem_2 = submod_1[0]
            getitem_3 = submod_1[1];  submod_1 = None
            submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
            return submod_2

    Output of split module is the same as output of input traced module.
    This is an example within a test setting:

        > orig_out = my_module_traced(x, y)
        > submodules_out = module_with_submodules(x, y)
        > self.assertEqual(orig_out, submodules_out)
        True
z%szpre split_moduleT)colorednodebase_mod_envbase_mod_attrsc                   > U R                   S:X  a  [        U R                  5      S:  a  U R                  S   O[        R                  R
                  nT(       aU  U[        R                  R
                  L a  SOU4nTR                  SU R                  UU R                  S9XR                  '   O1TR                  U R                  U R                  US9XR                  '   U R                  R                  5       XR                     l        X4$ U R                   S:X  a  TR                  U R                  5      XR                  '   U R                  R                  5       XR                     l        [        U R                  [        5      (       d!  [!        S[        U R                  5       35      e[#        TU R                  5      nXRU R                  '   X4$ )Nplaceholderr   r.   )args	type_expr)rJ   default_valueget_attrExpected str target, got )oplenrI   inspect	Signatureemptycreate_noder   typerH   targetmetacopyrL   
isinstancer,   AssertionErrorr9   )	rD   rE   rF   rK   rI   r7   base_mod_graphr@   r;   s	         r    construct_graph%split_module.<locals>.construct_graph   s   
 77m# #DII 2		!8I8I8O8O  ''7+<+<+B+BBBHX  +9*D*D!II"ii	 +E +YY' +9*D*DKK"ii"/ +E +YY'
 ,099>>+;L#( ++ WW
"&4&=&=dkk&JL#+/99>>+;L#(dkk3//$'@dkkAR@S%TUU.q$++>H*24;;'++r#   r   Ndef_nodeuse_nodec                   > SSK Jn  [        U SS 5      n[        USS 5      n[        R	                  SU R
                  UUb  UR
                  OSU5        X4:w  Ga  UbH  TU   nUR                  R                  U R
                  5        Ub  UR                  R                  U5        UGb4  TU   nUR                  R                  U R
                  5        U R                  R                  S5      =nb  [        U" U5      [        S9 H  nTU   n	UR                  R                  U	R
                  5        TU   R                  S:w  d  MB  [        U	SS 5      n
U
c  MT  TU
   nUR                  R                  U	R
                  5        UR                  R                  U5        UR                  R                  U
5        M     Ub  UR                  R                  U5        g g g g )	Nr   )free_symbols_fx_partitionz*record_cross_partition_use %s (%s) %s (%s)-example_value)keyrH   )%torch.fx.experimental.symbolic_shapesr`   r6   logdebugr   r   
setdefaultr   r   rV   getsortedr,   rN   r   )r]   r^   r`   defineduseddef_partitionuse_partitiondef_valss_node	s_defineds_def_partition
partitionssymbol_to_nodes               r    record_cross_partition_use0split_module.<locals>.record_cross_partition_use   s   F(OT:x$7		8MM%1HMMs	
 ?" *7 3%%00?#!,,77= *4 0$$//>  (}}00AAGN#L$9sC!/!2%,,77D)!,//=@ )0(NI(42<Y2G / 7 7 B B6;; O / : : E Ed K - : : E Ei P# D$ &!..99'B '3   r#   c                 8  > T" U 5      n[        U5      nTb  SR                  TU/5      n[        R                  SU R                  U5        TR                  U5      nUc  [        U5      =TU'   nUR                  R                  U R                  5        X l	        g )N_z*instantiate_node_partition_mapping %s (%s))
r,   joinrf   rg   r   ri   r   r   appendra   )rD   partition_idxpartition_name	partitionr:   rt   r=   s       r    "instantiate_node_partition_mapping8split_module.<locals>.instantiate_node_partition_mapping  s    &t,]+& !XX&GHN		8$))^	

 NN>2	5>~5NNJ~&##DII.+r#   rc   )rH   rL   outputcall_function   z*Expected 1 arg for _set_grad_enabled, got zExpected bool arg, got c              3   L   #    U  H  n[        U[        5      (       + v   M     g 7fN)rX   r
   .0args     r    	<genexpr>split_module.<locals>.<genexpr>U  s     J	z#t444	s   "$z3Expected all args to be python constants, not Nodesz'Expected 1 arg for _exit_autocast, got c              3   (   #    U  H  oS Lv   M
     g 7fr   r.   )r   vs     r    r   r   k  s     >&=}&=s   zautocast must exitzautocast_regions: %szgrad_regions: %s)rH   rL   r   c                    > T" U S 5      $ r   r.   )nrv   s    r    <lambda>split_module.<locals>.<lambda>  s    (B1d(Kr#   zSautocast or set_grad_enabled require monotonically increasing partitions: highest: z, this node's: c                    > T" U T5      $ r   r.   r]   rD   rv   s    r    r   r     s    ,FxQU,Vr#   c                    > T" U T5      $ r   r.   r   s    r    r   r     s    .HSW.Xr#   z cycle exists between partitions!z%Expected at least one region for nodec              3   $   #    U  H  ov   M     g 7fr   r.   r   s     r    r   r     s     8issis   )rN   rU   rI   kwargsrJ   c                     > T(       a  Tn O
ST 3n TS-  mTR                   R                  U TT   R                  S9nS TT'   U$ )Narg_r   )rJ   )r   rH   rT   )r   rH   counterinprA   
new_inputs
orig_nodesr~   s     r    add_placeholder%split_module.<locals>.add_placeholder  s\    +D "'+DqLG'oo99(o22 :  #'
3""r#   rL   rM   ra   c                    > TU    $ r   r.   r   r   s    r    r   r     s
    TUr#   c                    > TU    $ r   r.   r   s    r    r   r     s	    {1~r#   )call_modulerL   r2   ry   z&Expected tuple for gathered_args, got z'Expected dict for gathered_kwargs, got )rN   rU   rI   r   rJ   r   zExpected at least one regionzMissing exit noderH   c              3   H   >#    U  H  nTR                   TU      v   M     g 7fr   )r   )r   r   r   r~   s     r    r   r   Z  s%      
@QI!!*T"23@Qs   "r.   c              3   .   >#    U  H
  nTU   v   M     g 7fr   r.   )r   r   rE   s     r    r   r     s     B1A,t$1As   c                 "   > TU R                      $ r   )r   )r   rE   s    r    r   r     s    |AFF?Sr#   zpost split_module)Prf   rg   r   r
   dictr,   r   r   graph_moduler	   sympyr   amp_enter_autocast_exit_autocast_C_set_grad_enabledr   setr   nodesrV   ri   rX   SymIntSymFloatrD   exprSymbolrN   rU   rO   rI   rY   boolrT   alladdremovevaluesitemsrj   _LOGGERisEnabledForloggingDEBUGr   map_argr   listkeysr   r{   popr   RuntimeErrorr   rS   tuplerW   r   r9   nnModulerL   r   r4   ra   replacer   reversedr   r   r   r   proxyProxy	enumeratenextiter)Jr;   r<   r=   r>   r?   r@   rA   r:   r[   r   r   GLOBAL_STATE_NODESgrad_regionsautocast_regionsautocast_exitsactive_gradactive_autocastsvals0akr   assert_monotonically_increasinghighest_partitionpidoriginal_partition_orderroot_partitionsr}   sorted_partitionsroot_partition	dependentregions_mappingregionsrnew_node	orig_noder   	orig_attrrH   gathered_argsgathered_kwargsrU   target_attrr0   r   	exit_nodeorig_mod_envrF   construct_order_partitionsalready_constructed_attr_nodesrD   original_orderoutput_valsnum_output_valsrd   orig_mod_attr_nodes_based_mod_attrs
output_valnum_outputsoutput_val_proxyioutput_nameretrE   rZ   r   r   r   r   r   r~   rt   rv   ru   sJ   ` `  ```                                          `            @@@@@@@@@@@r    r   r   6   s   ` II11dC
!,!,39o!, S%(("7"7"C"CCD!, !,F ')J"$J/1N/CT /CXd^ /C /Cb,. 			!!		  "" 1<L 5@M13NKu IIMM/22S?3u~~ >??.2==.(,0N388==)77;;*4077o%$++9K*K{{ehh888tyy>Q&(DS^DTU  "$))A,55(+B4		RSCUBV)WXX",/1E0F,G)		 9 99J		JJJ(M  !$$T*),nT.B-C)D &'+t$		 8 88tyy>Q&(A#dii.AQR  !1.22>$3GH ''		!5/3tyy|,"%)).*>?!AQ##N4$89 "i n >n&;&;&=>>>122 2B1G1G1IJ1IA6!91IJ-9-?-?-AB-ATQAvayL-ALBGMM**,.>?(,7&*+;&<&R\@R#  $
499 771177hHHNN""		!K * &C 3&$,,=+>ocUT  !$ ;;00HHNN""		V HHNN""X9 @  $JOO$56!#O%/%5%5%7!	9))**"">2 &8
 $&
(,,.  0#N3>>Iy!..22>Bi(555&&y1 ? / Z0=>> -;,224MD'7|q $%LMM<@Js71:'33D9 QR[&s1v.	$??66ww;;8dii88"ii 7  IINN$  /7	%%d+ ! 5 <. ,~.	&(
##C"3I# # ||z)!)"2"2C88(3D9I9I4J3KL  4Ay7G7GH	i99"+//":":9;K;K"LK:CI%%i&6&67"1"3K-/)#3388:K5@I!!*S/2E $F &	S ,X 4))"4#5#56I $//K!HHNN22499>VWM#hhnn445O ww995aE,,S#6,7	!!&)+ #,"7"7!8&BH-1[[L*mU33$<T-=P<QR  ot44$=d?>S=TU  !8499TD 2277"&)) 3 H !IINN,HM*2I!!$'U Z --_-D%d+G7|q $%CDDSb\&s1v.	*40	$()<==$??66 ||$++#//57'nn 7  NN'')  " . .. %'L$&L+088>>+?+?+ANCENGGMMD+:lN,(L. " GGMMD&*L# " "5:R  &)U" ()ww}}Q}t=8Pd}NQ4~.	  
@I@Q@Q
 

 k*aOO"";q>2q OO"";/ OO""2& %++/+Cn, "S!+   / '991@,2.. /2248 ' ,99/>,0,n /2248 , 160E0E0Q0Qy1
y,,-
 $//!!B1A1ABB


 )++,?$xx~~33J?"+I,=,=">;,<Q,?,D,D[) #?A:DLd9#4#4567y 5D <GGMMD+:lN,(L. " 77h!!&&tyy|5ST  ((


+
+NN
KCII2CF Jw	 KBF R,/s*   AC%AC+:AC1AC1
AC6	AC6)NFFT) rP   r   collectionsr   collections.abcr   typingr   r   r   torch.fx._compatibilityr   torch.fx._utilsr   torch.fx.graph_moduler	   torch.fx.noder
   __all__	getLoggerr(   rf   r   r   r   r   r,   r9   intr   r   r   r.   r#   r    <module>r     s8     # $    1 2 -  
'!!(+ +g d+
 
 ,
0 C C  d+
 .2*/.3%)s	 &*s	s	HHOOs	 dVS[)s	 4S>*	s	
 "$s	 &d^s	 #s	 c]s	 ,s	r#   