
    Z jW              	          S r SSKrSSKrSSKJr  SSKrSSKJs  Jr	  SSKJr  SSK
Jr  SSKJr  SSKJrJr  SS	KJr  SS
KJrJr  SSKJr  \R2                  " \5      rS+S\R8                  S\S\S\R8                  4S jjr " S S\R@                  5      r! " S S\R@                  5      r" " S S\R@                  5      r# " S S\R@                  5      r$ " S S\R@                  5      r% " S S\R@                  5      r& " S S\R@                  5      r' " S  S!\R@                  5      r(\ " S" S#\5      5       r)\ " S$ S%\)5      5       r*\" S&S'9 " S( S)\)5      5       r+/ S*Qr,g),zPyTorch PVT model.    N)Iterable)nn   )initialization)ACT2FN)BaseModelOutputImageClassifierOutput)PreTrainedModel)auto_docstringlogging   )	PvtConfiginput	drop_probtrainingreturnc                    US:X  d  U(       d  U $ SU-
  nU R                   S   4SU R                  S-
  -  -   nU[        R                  " X@R                  U R
                  S9-   nUR                  5         U R                  U5      U-  nU$ )z[
Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

        r   r   )r   )dtypedevice)shapendimtorchrandr   r   floor_div)r   r   r   	keep_probr   random_tensoroutputs          u/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/pvt/modeling_pvt.py	drop_pathr!   &   s    
 CxII[[^

Q 77E

5ELL YYMYYy!M1FM    c                      ^  \ rS rSrSrSS\S-  SS4U 4S jjjrS\R                  S\R                  4S jr	S\
4S	 jrS
rU =r$ )PvtDropPath6   zXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr   r   c                 .   > [         TU ]  5         Xl        g N)super__init__r   )selfr   	__class__s     r    r)   PvtDropPath.__init__9   s    "r"   hidden_statesc                 B    [        XR                  U R                  5      $ r'   )r!   r   r   r*   r-   s     r    forwardPvtDropPath.forward=   s    FFr"   c                      SU R                    3$ )Nzp=r   )r*   s    r    
extra_reprPvtDropPath.extra_repr@   s    DNN#$$r"   r3   r'   )__name__
__module____qualname____firstlineno____doc__floatr)   r   Tensorr0   strr4   __static_attributes____classcell__r+   s   @r    r$   r$   6   sQ    b#%$, #$ # #GU\\ Gell G%C % %r"   r$   c                      ^  \ rS rSrSr SS\S\\\   -  S\\\   -  S\S\S\S	\4U 4S
 jjjr	S\
R                  S\S\S\
R                  4S jrS\
R                  S\\
R                  \\4   4S jrSrU =r$ )PvtPatchEmbeddingsD   z
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
config
image_size
patch_sizestridenum_channelshidden_size	cls_tokenc                   > [         T	U ]  5         Xl        [        U[        R
                  R                  5      (       a  UOX"4n[        U[        R
                  R                  5      (       a  UOX34nUS   US   -  US   US   -  -  nX l        X0l        XPl	        Xl
        [        R                  " [        R                  " SU(       a  US-   OUU5      5      U l        U(       a,  [        R                  " [        R                   " SSU5      5      OS U l        [        R$                  " XVXCS9U l        [        R(                  " XaR*                  S9U l        [        R.                  " UR0                  S9U l        g )Nr   r   kernel_sizerG   eps)p)r(   r)   rD   
isinstancecollectionsabcr   rE   rF   rH   num_patchesr   	Parameterr   randnposition_embeddingszerosrJ   Conv2d
projection	LayerNormlayer_norm_eps
layer_normDropouthidden_dropout_probdropout)
r*   rD   rE   rF   rG   rH   rI   rJ   rT   r+   s
            r    r)   PvtPatchEmbeddings.__init__K   s    	#-j+//:R:R#S#SZZdYq
#-j+//:R:R#S#SZZdYq
!!}
15*Q-:VW=:XY$$(&#%<<KKi;?[+V$
  JSekk!Q&DEX\))L6e,,{8M8MNzzF$>$>?r"   
embeddingsheightwidthr   c                    X#-  n[         R                  R                  5       (       d<  X@R                  R                  U R                  R                  -  :X  a  U R
                  $ UR                  SX#S5      R                  SSSS5      n[        R                  " XU4SS9nUR                  SSX#-  5      R                  SSS5      nU$ )Nr   r   r      bilinear)sizemode)
r   jit
is_tracingrD   rE   rW   reshapepermuteFinterpolate)r*   rb   rc   rd   rT   interpolated_embeddingss         r    interpolate_pos_encoding+PvtPatchEmbeddings.interpolate_pos_encodingg   s    n yy##%%+9O9ORVR]R]RhRh9h*h+++''6"=EEaAqQ
"#--
%Wa"b"9"A"A!R"X"`"`abdegh"i&&r"   pixel_valuesc                 ~   UR                   u  p#pEX0R                  :w  a  [        S5      eU R                  U5      nUR                   Gt ptnUR	                  S5      R                  SS5      nU R                  U5      nU R                  b  U R                  R                  USS5      n	[        R                  " X4SS9nU R                  U R                  S S 2SS 24   XE5      n
[        R                  " U R                  S S 2S S24   U
4SS9n
OU R                  U R                  XE5      n
U R                  X-   5      nXU4$ )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rg   r   rf   dim)r   rH   
ValueErrorrZ   flatten	transposer]   rJ   expandr   catrr   rW   r`   )r*   rt   
batch_sizerH   rc   rd   patch_embed_rb   rJ   rW   s              r    r0   PvtPatchEmbeddings.forwardr   s8   2>2D2D/
&,,,w  ool3'--E!))!,66q!<__[1
>>%--j"bAII#:BJ"&"?"?@X@XYZ\]\^Y^@_ag"o"'))T-E-Ea!e-LNa,bhi"j"&"?"?@X@XZ`"h\\*"BC
5((r"   )
rJ   rD   r`   rE   r]   rH   rT   rF   rW   rZ   F)r6   r7   r8   r9   r:   r   intr   boolr)   r   r<   rr   tupler0   r>   r?   r@   s   @r    rB   rB   D   s      @@ (3-'@ (3-'	@
 @ @ @ @ @8	'5<< 	' 	'UX 	']b]i]i 	')ELL )U5<<c;Q5R ) )r"   rB   c                   n   ^  \ rS rSrS\S\4U 4S jjrS\R                  S\R                  4S jr	Sr
U =r$ )	PvtSelfOutput   rD   rI   c                    > [         TU ]  5         [        R                  " X"5      U l        [        R
                  " UR                  5      U l        g r'   )r(   r)   r   Lineardenser^   r_   r`   )r*   rD   rI   r+   s      r    r)   PvtSelfOutput.__init__   s4    YY{8
zz&"<"<=r"   r-   r   c                 J    U R                  U5      nU R                  U5      nU$ r'   r   r`   r/   s     r    r0   PvtSelfOutput.forward   s$    

=1]3r"   r   r6   r7   r8   r9   r   r   r)   r   r<   r0   r>   r?   r@   s   @r    r   r      s6    >y >s >
U\\ ell  r"   r   c                      ^  \ rS rSrSrS\S\S\S\4U 4S jjrS\S	\	R                  4S
 jr SS\	R                  S\S\S\S	\\	R                     4
S jjrSrU =r$ )PvtEfficientSelfAttention   zxEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://huggingface.co/papers/2102.12122).rD   rI   num_attention_headssequences_reduction_ratioc                 ~  > [         TU ]  5         X l        X0l        U R                  U R                  -  S:w  a&  [	        SU R                   SU R                   S35      e[        U R                  U R                  -  5      U l        U R                  U R                  -  U l        [        R                  " U R                  U R                  UR                  S9U l        [        R                  " U R                  U R                  UR                  S9U l        [        R                  " U R                  U R                  UR                  S9U l        [        R                  " UR                  5      U l        X@l        US:  a>  [        R$                  " X"XDS9U l        [        R(                  " X!R*                  S9U l        g g )	Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ())biasr   rL   rN   )r(   r)   rI   r   rx   r   attention_head_sizeall_head_sizer   r   qkv_biasquerykeyvaluer^   attention_probs_dropout_probr`   r   rY   sequence_reductionr[   r\   r]   r*   rD   rI   r   r   r+   s        r    r)   "PvtEfficientSelfAttention.__init__   se    	&#6 d666!;#D$4$4#5 622316 
 $'t'7'7$:R:R'R#S !558P8PPYYt//1C1C&//Z
99T--t/A/AXYYt//1C1C&//Z
zz&"E"EF)B&$q(&(ii6O'D# !ll;<Q<QRDO	 )r"   r-   r   c                     UR                  5       S S U R                  U R                  4-   nUR                  U5      nUR	                  SSSS5      $ )Nrf   r   rg   r   r   )ri   r   r   viewrn   )r*   r-   	new_shapes      r    transpose_for_scores.PvtEfficientSelfAttention.transpose_for_scores   sT    !&&("-1I1I4KcKc0dd	%**95$$Q1a00r"   rc   rd   output_attentionsc                    U R                  U R                  U5      5      nU R                  S:  aw  UR                  u  pgnUR	                  SSS5      R                  XhX#5      nU R                  U5      nUR                  XhS5      R	                  SSS5      nU R                  U5      nU R                  U R                  U5      5      n	U R                  U R                  U5      5      n
[        R                  " XYR                  SS5      5      nU[        R                  " U R                  5      -  n[         R"                  R%                  USS9nU R'                  U5      n[        R                  " X5      nUR	                  SSSS5      R)                  5       nUR+                  5       S S U R,                  4-   nUR/                  U5      nU(       a  X4nU$ U4nU$ )Nr   r   rg   rf   rv   r   )r   r   r   r   rn   rm   r   r]   r   r   r   matmulrz   mathsqrtr   r   
functionalsoftmaxr`   
contiguousri   r   r   )r*   r-   rc   rd   r   query_layerr}   seq_lenrH   	key_layervalue_layerattention_scoresattention_probscontext_layernew_context_layer_shapeoutputss                   r    r0   !PvtEfficientSelfAttention.forward   s    //

=0IJ))A-0=0C0C-J)11!Q:BB:]ckM 33MBM)11*BOWWXY[\^_`M OOM:M--dhh}.EF	//

=0IJ !<<5H5HR5PQ+dii8P8P.QQ --//0@b/I ,,7_B%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=2 O\M]r"   )r   r   r`   rI   r   r]   r   r   r   r   r   r   )r6   r7   r8   r9   r:   r   r   r;   r)   r   r<   r   r   r   r0   r>   r?   r@   s   @r    r   r      s     CSS.1SHKShmS:1# 1%,, 1 #(*||* * 	*
  * 
u||	* *r"   r   c                      ^  \ rS rSrS\S\S\S\4U 4S jjr SS\R                  S\S	\S
\
S\\R                     4
S jjrSrU =r$ )PvtAttention   rD   rI   r   r   c                 `   > [         TU ]  5         [        UUUUS9U l        [	        XS9U l        g )N)rI   r   r   )rI   )r(   r)   r   r*   r   r   r   s        r    r)   PvtAttention.__init__   s6     	-# 3&?	
	 $FDr"   r-   rc   rd   r   r   c                 d    U R                  XX45      nU R                  US   5      nU4USS  -   nU$ )Nr   r   )r*   r   )r*   r-   rc   rd   r   self_outputsattention_outputr   s           r    r0   PvtAttention.forward   s@     yyQ;;|A7#%QR(88r"   )r   r*   r   )r6   r7   r8   r9   r   r   r;   r)   r   r<   r   r   r0   r>   r?   r@   s   @r    r   r      ss    
E
E.1
EHK
Ehm
E _d"\\36?BW[	u||	 r"   r   c            
          ^  \ rS rSr  SS\S\S\S-  S\S-  4U 4S jjjrS\R                  S	\R                  4S
 jr	Sr
U =r$ )PvtFFN   NrD   in_featureshidden_featuresout_featuresc                 x  > [         TU ]  5         Ub  UOUn[        R                  " X#5      U l        [        UR                  [        5      (       a  [        UR                     U l	        OUR                  U l	        [        R                  " X45      U l
        [        R                  " UR                  5      U l        g r'   )r(   r)   r   r   dense1rQ   
hidden_actr=   r   intermediate_act_fndense2r^   r_   r`   )r*   rD   r   r   r   r+   s        r    r)   PvtFFN.__init__   s     	'3'?|[ii=f''--'-f.?.?'@D$'-'8'8D$ii>zz&"<"<=r"   r-   r   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ r'   )r   r   r`   r   r/   s     r    r0   PvtFFN.forward  sP    M200?]3M2]3r"   )r   r   r`   r   )NNr   r@   s   @r    r   r      sc    
 '+#'>> > t	>
 Dj> >"U\\ ell  r"   r   c                   v   ^  \ rS rSrS\S\S\S\S\S\4U 4S jjrSS	\R                  S
\S\S\
4S jjrSrU =r$ )PvtLayeri  rD   rI   r   r!   r   	mlp_ratioc                 ^  > [         TU ]  5         [        R                  " X!R                  S9U l        [        UUUUS9U l        US:  a  [        U5      O[        R                  " 5       U l
        [        R                  " X!R                  S9U l        [        X&-  5      n[        XUS9U l        g )NrN   )rD   rI   r   r   r   )rD   r   r   )r(   r)   r   r[   r\   layer_norm_1r   	attentionr$   Identityr!   layer_norm_2r   r   mlp)	r*   rD   rI   r   r!   r   r   mlp_hidden_sizer+   s	           r    r)   PvtLayer.__init__  s     	LL:O:OP%# 3&?	
 4=s?Y/LL:O:OPk56Rabr"   r-   rc   rd   r   c                     U R                  U R                  U5      UUUS9nUS   nUSS  nU R                  U5      nXa-   nU R                  U R	                  U5      5      nU R                  U5      nX-   n	U	4U-   nU$ )N)r-   rc   rd   r   r   r   )r   r   r!   r   r   )
r*   r-   rc   rd   r   self_attention_outputsr   r   
mlp_outputlayer_outputs
             r    r0   PvtLayer.forward/  s    !%++M:/	 "0 "
 2!4(,>>*:;(8XXd//>?
^^J/
$1/G+r"   )r   r!   r   r   r   r   )r6   r7   r8   r9   r   r   r;   r)   r   r<   r   r0   r>   r?   r@   s   @r    r   r     st    cc c !	c
 c $)c c,U\\ 3 s _c  r"   r   c                      ^  \ rS rSrS\4U 4S jjr   SS\R                  S\S-  S\S-  S\S-  S	\	\
-  4
S
 jjrSrU =r$ )
PvtEncoderiF  rD   c                   > [         T	U ]  5         Xl        [        R                  " SUR
                  [        UR                  5      SS9R                  5       n/ n[        UR                  5       H  nUR                  [        UUS:X  a  UR                  OU R                  R                  SUS-   -  -  UR                  U   UR                  U   US:X  a  UR                   OUR"                  US-
     UR"                  U   XAR                  S-
  :H  S95        M     [$        R&                  " U5      U l        / nSn[        UR                  5       H  n/ nUS:w  a  XaR                  US-
     -  n[        UR                  U   5       HY  nUR                  [+        UUR"                  U   UR,                  U   X&U-      UR.                  U   UR0                  U   S95        M[     UR                  [$        R&                  " U5      5        M     [$        R&                  " U5      U l        [$        R4                  " UR"                  S   UR6                  S	9U l        g )
Nr   cpu)r   rg   r   )rD   rE   rF   rG   rH   rI   rJ   )rD   rI   r   r!   r   r   rf   rN   )r(   r)   rD   r   linspacedrop_path_ratesumdepthstolistrangenum_encoder_blocksappendrB   rE   patch_sizesstridesrH   hidden_sizesr   
ModuleListpatch_embeddingsr   r   sequence_reduction_ratios
mlp_ratiosblockr[   r\   r]   )
r*   rD   drop_path_decaysrb   iblockscurlayersjr+   s
            r    r)   PvtEncoder.__init__G  s    !>>!V-B-BCDV_delln 
v001A"!45Fv00@V@V[\abefaf[g@h%11!4!>>!,89Q!4!4FDWDWXY\]X]D^ & 3 3A 6#<#<q#@@
 2 !#j 9 v001AFAv}}QU++6==+,%$*$7$7$:,2,F,Fq,I"27";282R2RST2U"("3"3A"6	 - MM"--/0! 2$ ]]6*
 ,,v':':2'>FDYDYZr"   rt   r   Noutput_hidden_statesreturn_dictr   c                 d   U(       a  SOS nU(       a  SOS nUR                   S   n[        U R                  5      nUn	[        [	        U R
                  U R                  5      5       H  u  n
u  pU" U	5      u  pnU H/  nU" XX5      nUS   n	U(       a	  UUS   4-   nU(       d  M*  XY4-   nM1     XS-
  :w  d  MR  U	R                  X}US5      R                  SSSS5      R                  5       n	M     U R                  U	5      n	U(       a  XY4-   nU(       d  [        S XU4 5       5      $ [        U	UUS9$ )	N r   r   rf   r   rg   c              3   .   #    U  H  oc  M  Uv   M     g 7fr'   r   ).0vs     r    	<genexpr>%PvtEncoder.forward.<locals>.<genexpr>  s     m$[q$[s   	last_hidden_stater-   
attentions)r   lenr   	enumeratezipr   rm   rn   r   r]   r   r   )r*   rt   r   r   r   all_hidden_statesall_self_attentionsr}   
num_blocksr-   idxembedding_layerblock_layerrc   rd   r   layer_outputss                    r    r0   PvtEncoder.forwardy  sK    #7BD$5b4!''*
_
$3<SAVAVX\XbXb=c3d/C//+:=+I(M5$ %mU V -a 0$*=qAQ@S*S'''(9<L(L% % 1n$ - 5 5j%QS T \ \]^`acdfg h s s u 4e 6 14D Dm]GZ$[mmm++*
 	
r"   )r   rD   r]   r   )FFT)r6   r7   r8   r9   r   r)   r   FloatTensorr   r   r   r0   r>   r?   r@   s   @r    r   r   F  sn    0[y 0[j */,1#'#
''#
  $;#
 #Tk	#

 D[#
 
	 #
 #
r"   r   c                   ~    \ rS rSr% \\S'   SrSrSr/ r	\
R                  " 5       S\R                  SS4S	 j5       rS
rg)PvtPreTrainedModeli  rD   pvtrt   )imagemoduler   Nc                    U R                   R                  n[        U[        R                  [        R
                  45      (       aO  [        R                  " UR                  SUS9  UR                  b!  [        R                  " UR                  5        gg[        U[        R                  5      (       aA  [        R                  " UR                  5        [        R                  " UR                  5        g[        U[        5      (       aO  [        R                  " UR                  SUS9  UR                  b!  [        R                  " UR                  SUS9  ggg)zInitialize the weightsr   )meanstdN)rD   initializer_rangerQ   r   r   rY   inittrunc_normal_weightr   zeros_r[   ones_rB   rW   rJ   )r*   r  r  s      r    _init_weights PvtPreTrainedModel._init_weights  s     kk++fryy"))455v}}3C@{{&FKK( '--KK$JJv}}% 233v99M+""6#3#3#3G , 4r"   r   )r6   r7   r8   r9   r   __annotations__base_model_prefixmain_input_nameinput_modalities_no_split_modulesr   no_gradr   Moduler"  r>   r   r"   r    r  r    sM    $O!
]]_HBII H$ H Hr"   r  c                      ^  \ rS rSrS\4U 4S jjr\   SS\R                  S\	S-  S\	S-  S\	S-  S	\
\-  4
S
 jj5       rSrU =r$ )PvtModeli  rD   c                 p   > [         TU ]  U5        Xl        [        U5      U l        U R                  5         g r'   )r(   r)   rD   r   encoder	post_initr*   rD   r+   s     r    r)   PvtModel.__init__  s/      "&) 	r"   Nrt   r   r   r   r   c                 0   Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nU R	                  UUUUS9nUS   nU(       d	  U4USS  -   $ [        UUR                  UR                  S9$ )Nrt   r   r   r   r   r   r  )rD   r   r   r   r.  r   r-   r  )r*   rt   r   r   r   kwargsencoder_outputssequence_outputs           r    r0   PvtModel.forward  s     2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY,,%/!5#	 ' 
 *!,#%(;;;-)77&11
 	
r"   )rD   r.  )NNN)r6   r7   r8   r9   r   r)   r   r   r  r   r   r   r0   r>   r?   r@   s   @r    r,  r,    su    y   *.,0#'
''
  $;
 #Tk	

 D[
 
	 
 
r"   r,  z
    Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    )custom_introc                      ^  \ rS rSrS\SS4U 4S jjr\    SS\R                  S-  S\R                  S-  S\	S-  S	\	S-  S
\	S-  S\
\-  4S jj5       rSrU =r$ )PvtForImageClassificationi  rD   r   Nc                 6  > [         TU ]  U5        UR                  U l        [        U5      U l        UR                  S:  a.  [
        R                  " UR                  S   UR                  5      O[
        R                  " 5       U l	        U R                  5         g )Nr   rf   )r(   r)   
num_labelsr,  r  r   r   r   r   
classifierr/  r0  s     r    r)   "PvtForImageClassification.__init__  sy      ++F# FLEVEVYZEZBIIf))"-v/@/@A`b`k`k`m 	
 	r"   rt   labelsr   r   r   c                 T   Ub  UOU R                   R                  nU R                  UUUUS9nUS   nU R                  USS2SSS24   5      n	Sn
Ub  U R	                  X)U R                   5      n
U(       d  U	4USS -   nU
b  U
4U-   $ U$ [        U
U	UR                  UR                  S9$ )ab  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nr3  r   r   )losslogitsr-   r  )rD   r   r  r=  loss_functionr	   r-   r  )r*   rt   r?  r   r   r   r4  r   r6  rB  rA  r   s               r    r0   !PvtForImageClassification.forward  s      &1%<k$++BYBY((%/!5#	  
 "!*Aq!9:%%fdkkBDY,F)-)9TGf$EvE$!//))	
 	
r"   )r=  r<  r  )NNNN)r6   r7   r8   r9   r   r)   r   r   r<   r   r   r	   r0   r>   r?   r@   s   @r    r:  r:    s    y T   '+)-,0#')
llT))
 t#)
  $;	)

 #Tk)
 D[)
 
&	&)
 )
r"   r:  )r:  r,  r  )r   F)-r:   rR   r   collections.abcr   r   torch.nn.functionalr   r   ro    r   r  activationsr   modeling_outputsr   r	   modeling_utilsr
   utilsr   r   configuration_pvtr   
get_loggerr6   loggerr<   r;   r   r!   r*  r$   rB   r   r   r   r   r   r   r  r,  r:  __all__r   r"   r    <module>rP     sc       $     & ! F - , ( 
		H	%U\\ e T V[VbVb  %")) %A) A)H	BII 	O		 Od299 .RYY 6+ryy +\V
 V
r H H H0 )
! )
 )
X 9
 2 9
9
x Jr"   