
    Z j              	       4   S r SSKrSSKrSSKJr  SSKrSSKJrJr  SSK	J
r
  SSKJr  SSKJr  SS	KJrJr  SS
KJr  SSKJrJrJrJrJrJr  SSKJr  SSKJr  SSK J!r!J"r"J#r#  SSK$J%r%  SSK&J'r'  \"RP                  " \)5      r*\\!" SS9 " S S\5      5       5       r+SPS\R                  S\,S\-S\R                  4S jjr. " S S\R^                  5      r0 " S S\R^                  5      r1 " S S \R^                  5      r2 " S! S"\R^                  5      r3 " S# S$\35      r4 " S% S&\R^                  5      r5\3\4S'.r6 " S( S)\R^                  5      r7 " S* S+\R^                  5      r8 " S, S-\R^                  5      r9 " S. S/\5      r: " S0 S1\R^                  5      r; " S2 S3\R^                  5      r<\! " S4 S5\5      5       r=\! " S6 S7\=5      5       r> " S8 S9\R^                  5      r?\!" S:S9 " S; S<\=5      5       r@\!" S=S9 " S> S?\=5      5       rA " S@ SA\R^                  5      rB " SB SC\R^                  5      rC " SD SE\R^                  5      rD " SF SG\R^                  5      rE " SH SI\R^                  5      rF\! " SJ SK\=5      5       rG\!" SLS9 " SM SN\\=5      5       rH/ SOQrIg)QzPyTorch BEiT model.    N)	dataclass)Tensornn)CrossEntropyLoss   )initialization)ACT2FN)BackboneMixinfilter_output_hidden_states)GradientCheckpointingLayer)BackboneOutputBaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputMaskedLMOutputSemanticSegmenterOutput)PreTrainedModel)#compile_compatible_method_lru_cache)auto_docstringlogging	torch_int)can_return_tuple   )
BeitConfigz-
    Class for outputs of [`BeitModel`].
    )custom_introc                       \ rS rSrSrSrg)BeitModelOutputWithPooling.   a2  
pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
    Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
    *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
    will be returned.
 N)__name__
__module____qualname____firstlineno____doc____static_attributes__r       w/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/beit/modeling_beit.pyr   r   .   s    r&   r   input	drop_probtrainingreturnc                    US:X  d  U(       d  U $ SU-
  nU R                   S   4SU R                  S-
  -  -   nU[        R                  " X@R                  U R
                  S9-   nUR                  5         U R                  U5      U-  nU$ )z[
Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

        r   r   )r   )dtypedevice)shapendimtorchrandr.   r/   floor_div)r(   r)   r*   	keep_probr0   random_tensoroutputs          r'   	drop_pathr9   =   s    
 CxII[[^

Q 77E

5ELL YYMYYy!M1FMr&   c                      ^  \ rS rSrSrSS\S-  SS4U 4S jjjrS\R                  S\R                  4S jr	S\
4S	 jrS
rU =r$ )BeitDropPathL   zXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr)   r+   c                 .   > [         TU ]  5         Xl        g N)super__init__r)   )selfr)   	__class__s     r'   r@   BeitDropPath.__init__O   s    "r&   hidden_statesc                 B    [        XR                  U R                  5      $ r>   )r9   r)   r*   rA   rD   s     r'   forwardBeitDropPath.forwardS   s    FFr&   c                      SU R                    3$ )Nzp=r)   rA   s    r'   
extra_reprBeitDropPath.extra_reprV   s    DNN#$$r&   rJ   r>   )r    r!   r"   r#   r$   floatr@   r2   r   rG   strrL   r%   __classcell__rB   s   @r'   r;   r;   L   sQ    b#%$, #$ # #GU\\ Gell G%C % %r&   r;   c                      ^  \ rS rSrSrS\SS4U 4S jjrS\R                  S\	S	\	S\R                  4S
 jr
 SS\R                  S\R                  S-  S\R                  4S jjrSrU =r$ )BeitEmbeddings\   z[
Construct the CLS token, position and patch embeddings. Optionally, also the mask token.

configr+   Nc                 ^  > [         TU ]  5         [        R                  " [        R
                  " SSUR                  5      5      U l        UR                  (       a<  [        R                  " [        R
                  " SSUR                  5      5      U l	        OS U l	        [        U5      U l        UR                  U l        [        UR                  [        R                   R"                  5      (       a  UR                  OUR                  UR                  4U l        U R                  R$                  nUR&                  (       a?  [        R                  " [        R
                  " SUS-   UR                  5      5      U l        OS U l        [        R*                  " UR,                  5      U l        g )Nr   )r?   r@   r   	Parameterr2   zeroshidden_size	cls_tokenuse_mask_token
mask_tokenBeitPatchEmbeddingspatch_embeddings
patch_size
isinstance
image_sizecollectionsabcIterablenum_patches use_absolute_position_embeddingsposition_embeddingsDropouthidden_dropout_probdropout)rA   rU   re   rB   s      r'   r@   BeitEmbeddings.__init__b   s'   ekk!Q8J8J&KL   ll5;;q!V=O=O+PQDO"DO 3F ; ++ &++[__-E-EFF ##V%6%67 	
 ++7722')||EKK;QR?TZTfTf4g'hD$'+D$zz&"<"<=r&   
embeddingsheightwidthc                    UR                   S   S-
  nU R                  R                   S   S-
  n[        R                  R	                  5       (       d  XE:X  a  X#:X  a  U R                  $ U R                  SS2SS24   nU R                  SS2SS24   nUR                   S   nX R
                  -  n	X0R
                  -  n
[        US-  5      nUR                  SXU5      nUR                  SSSS5      n[        R                  R                  UX4SS	S
9nUR                  SSSS5      R                  SSU5      n[        R                  " Xg4SS9$ )a  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
r   Ng      ?r   r      bicubicFsizemodealign_cornersdim)r0   rg   r2   jit
is_tracingr_   r   reshapepermuter   
functionalinterpolateviewcat)rA   rl   rm   rn   re   num_positionsclass_pos_embedpatch_pos_embedrx   
new_height	new_widthsqrt_num_positionss               r'   interpolate_pos_encoding'BeitEmbeddings.interpolate_pos_encodingy   sS    !&&q)A-0066q9A= yy##%%+*F6?+++221bqb59221ab59r".
__,	&}c'9:)11!5G]`a)11!Q1=--33(	 4 
 *11!Q1=BB1b#Nyy/;CCr&   pixel_valuesbool_masked_posc                    UR                   u    p4nU R                  U5      u  nu  pxUR                  5       u  pnUbI  U R                  R	                  XS5      nUR                  S5      R                  U5      nUSU-
  -  X-  -   nU R                  R	                  U	SS5      n[        R                  " X4SS9nU R                  b  X`R                  XdU5      -   nU R                  U5      nXgU44$ Nrp   r   rw   )r0   r^   rt   r\   expand	unsqueezetype_asrZ   r2   r   rg   r   rj   )rA   r   r   _rm   rn   rl   patch_heightpatch_width
batch_sizeseq_lenmask_tokensw
cls_tokenss                 r'   rG   BeitEmbeddings.forward   s    
 +001e262G2G2U/
/\!+!2
Q&//00bIK))"-55kBA#q1u-?J^^**:r2>
YY
7Q?
##/#&C&CJX]&^^J\\*-
+666r&   )rZ   rj   ra   r\   r^   r_   rg   r>   )r    r!   r"   r#   r$   r   r@   r2   r   intr   
BoolTensorrG   r%   rP   rQ   s   @r'   rS   rS   \   s    
>z >d >.&D5<< &D &DUX &D]b]i]i &DV 487ll7 ))D07 
	7 7r&   rS   c                   f   ^  \ rS rSrSrU 4S jrS\R                  S\R                  4S jrSr	U =r
$ )r]      z
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
c                    > [         TU ]  5         UR                  UR                  p2UR                  UR
                  pT[        U[        R                  R                  5      (       a  UOX"4n[        U[        R                  R                  5      (       a  UOX34nUS   US   -  US   US   -  -  nUS   US   -  US   US   -  4nX l        X0l        X@l        X`l
        Xpl        [        R                  " XEX3S9U l        g )Nr   r   kernel_sizestride)r?   r@   ra   r_   num_channelsrY   r`   rb   rc   rd   re   patch_shaper   Conv2d
projection)	rA   rU   ra   r_   r   rY   re   r   rB   s	           r'   r@   BeitPatchEmbeddings.__init__   s    !'!2!2F4E4EJ$*$7$79K9Kk#-j+//:R:R#S#SZZdYq
#-j+//:R:R#S#SZZdYq
!!}
15*Q-:VW=:XY!!}
15z!}
ST7UV$$(&&))L:ir&   r   r+   c                 V   UR                   u  p#pEX0R                  :w  a  [        S5      eU R                  UR	                  U R                  R
                  R                  5      5      nUR                   S   UR                   S   pUR                  S5      R                  SS5      nXgU44$ )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rq   r   r   )	r0   r   
ValueErrorr   toweightr.   flatten	transpose)	rA   r   r   r   rm   rn   rl   r   r   s	            r'   rG   BeitPatchEmbeddings.forward   s    2>2D2D/
&,,,w  __\__T__5K5K5Q5Q%RS
$.$4$4Q$79I9I!9Lk''*44Q:
+666r&   )ra   r   re   r   r_   r   )r    r!   r"   r#   r$   r@   r2   r   rG   r%   rP   rQ   s   @r'   r]   r]      s.    j"7ELL 7U\\ 7 7r&   r]   c                      ^  \ rS rSrSS\S\S-  SS4U 4S jjjr    SS\R                  S\	S	\R                  S-  S
\	S\\
   S-  S\\R                     \\R                  \R                  4   -  4S jjrSrU =r$ )BeitSelfAttention   NrU   window_sizer+   c                 J  > [         TU ]  5         Xl        UR                  UR                  -  S:w  a7  [        US5      (       d&  [        SUR                   SUR                   S35      eUR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l	        [        R                  " UR                  U R                  5      U l        [        R                  " UR                  U R                  SS9U l        [        R                  " UR                  U R                  5      U l        [        R                  " UR                   5      U l        [%        U5      U l        U R&                  (       a  [)        XS9U l        g g )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .F)biasr   )r?   r@   rU   rY   num_attention_headshasattrr   r   attention_head_sizeall_head_sizer   Linearquerykeyvaluerh   attention_probs_dropout_probrj   boolhas_relative_position_biasBeitRelativePositionBiasrelative_position_biasrA   rU   r   rB   s      r'   r@   BeitSelfAttention.__init__   sN    : ::a?PVXhHiHi"6#5#5"6 7334A7 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1C%PYYv1143E3EF
zz&"E"EF*.{*;'***B6*cD' +r&   rD   output_attentionsr   r   
resolutionc                     UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
[        R                  " XR	                  SS5      5      nU[        R                  " U R                  5      -  nU R                  (       aS  Uu  pXR                  R                  -  XR                  R                  -  4nXR                  XUR                   S   S9-   nUb  X-   n[        R                   R#                  USS9nU R%                  U5      n[        R                  " X5      nUR'                  SSSS5      R)                  5       nUR+                  5       S S U R,                  4-   nUR                  " U6 nU(       a  UU4nU$ U4nU$ )	Nrp   r   rq   dim_sizerw   r   r   )r0   r   r   r   r   r   r   r2   matmulmathsqrtr   rU   r_   r   r   r}   softmaxrj   r|   
contiguousrt   r   )rA   rD   r   r   r   r   input_shapehidden_shapequery_layer	key_layervalue_layerattention_scoresrm   rn   r   attention_probscontext_layernew_context_layer_shapeoutputss                      r'   rG   BeitSelfAttention.forward   s     $))#2.CCbC$*B*BCjj/44\BLLQPQRHH]+00>HHAN	jj/44\BLLQPQR !<<5H5HR5PQ+dii8P8P.QQ **&MF![[%;%;;UkkF\F\=\]K/2M2M@S@STU@V 3N 3  
 "-/H --//0@b/I ,,7_B%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**,CD6G=/2 O\M]r&   )
r   r   rU   rj   r   r   r   r   r   r   r>   FNFNr    r!   r"   r#   r   tupler@   r2   r   r   r   rG   r%   rP   rQ   s   @r'   r   r      s    dz d dPT d d4 #(6:).(,.||.  . !&t 3	.
 #'. #J%. 
u||	uU\\5<<%?@	@. .r&   r   c                       \ rS rSr    SS\R
                  S\S\R
                  S-  S\S\\   S-  S\\R
                     \\R
                  \R
                  4   -  4S	 jjr	S
r
g)BeitSdpaSelfAttentioni*  NrD   r   r   r   r   r+   c           
      H   U(       a,  [         R                  U R                  R                   S35        UR                  S S n/ UQSPU R
                  P7nU R                  U5      R                  U5      R                  SS5      nU R                  U5      R                  U5      R                  SS5      n	U R                  U5      R                  U5      R                  SS5      n
S nU R                  (       aQ  Uu  pXR                  R                  -  XR                  R                  -  4nU R                  XUR                  S   S9nUb
  Uc  UnOX-  nS[        R                   " U R
                  5      -  n["        R$                  R&                  R)                  UU	U
UU R*                  (       a  U R                  R,                  OSSUS9nUR/                  S	SSS
5      R1                  5       nUR3                  5       S S U R4                  4-   nUR                  " U6 nUS 4$ )Nz does not support `output_attentions=True`. The returned attention weights will be `None`. If you want to get attention weights, please set `attn_implementation='eager'` when loading the model.rp   r   rq   r   r-   F)	attn_mask	dropout_p	is_causalscaler   r   r   )loggerwarning_oncerB   r    r0   r   r   r   r   r   r   r   rU   r_   r   r   r   r2   r   r}   scaled_dot_product_attentionr*   r   r|   r   rt   r   )rA   rD   r   r   r   r   r   r   r   r   r   	attn_biasrm   rn   r   scalingr   r   s                     r'   rG   BeitSdpaSelfAttention.forward+  s    >>**+ ,D D $))#2.CCbC$*B*BCjj/44\BLLQPQRHH]+00>HHAN	jj/44\BLLQPQR	**&MF![[%;%;;UkkF\F\=\]K33@S@STU@V 4 I
 "- 2	3	dii 8 899++HHBF--dkk>>UX I 
 &--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**,CDd""r&   r   r   )r    r!   r"   r#   r2   r   r   r   r   rG   r%   r   r&   r'   r   r   *  s     #(6:).(,/#||/#  /# !&t 3	/#
 #'/# #J%/# 
u||	uU\\5<<%?@	@/# /#r&   r   c                      ^  \ rS rSrSrS\SS4U 4S jjrSS\R                  S\R                  S\R                  4S	 jjr	S
r
U =r$ )BeitSelfOutputi]  z
The residual connection is defined in BeitLayer instead of here (as is the case with other models), due to the
layernorm applied before each block.
rU   r+   Nc                    > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  5      U l        g r>   )	r?   r@   r   r   rY   denserh   ri   rj   rA   rU   rB   s     r'   r@   BeitSelfOutput.__init__c  sB    YYv1163E3EF
zz&"<"<=r&   rD   input_tensorc                 J    U R                  U5      nU R                  U5      nU$ r>   r   rj   )rA   rD   r   gammas       r'   rG   BeitSelfOutput.forwardh  $    

=1]3r&   r   r>   )r    r!   r"   r#   r$   r   r@   r2   r   rG   r%   rP   rQ   s   @r'   r   r   ]  sI    
>z >d >
U\\  ^c^j^j  r&   r   )eagersdpac                      ^  \ rS rSrSS\S\S-  SS4U 4S jjjr    SS\R                  S\	S	\R                  S-  S
\	S\\
   S-  S\\R                     \\R                  \R                  4   -  4S jjrSrU =r$ )BeitAttentioniu  NrU   r   r+   c                 z   > [         TU ]  5         [        UR                     " XS9U l        [        U5      U l        g )Nr   )r?   r@   BEIT_SELF_ATTENTION_CLASSES_attn_implementation	attentionr   r8   r   s      r'   r@   BeitAttention.__init__v  s2    4V5P5PQRXr$V,r&   rD   r   r   r   r   c                 h    U R                  XX4U5      nU R                  US   U5      nU4USS  -   nU$ )Nr   r   r   r8   )	rA   rD   r   r   r   r   self_outputsattention_outputr   s	            r'   rG   BeitAttention.forward{  sK     ~~.D`j
  ;;|AF#%QR(88r&   r  r>   r   r   rQ   s   @r'   r   r   u  s    -z - -PT - - #(6:).(,||   !&t 3	
 #' #J% 
u||	uU\\5<<%?@	@ r&   r   c                   n   ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )	BeitIntermediatei  rU   r+   Nc                   > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g r>   )r?   r@   r   r   rY   intermediate_sizer   r`   
hidden_actrO   r	   intermediate_act_fnr   s     r'   r@   BeitIntermediate.__init__  s`    YYv1163K3KL
f''--'-f.?.?'@D$'-'8'8D$r&   rD   c                 J    U R                  U5      nU R                  U5      nU$ r>   r   r
  rF   s     r'   rG   BeitIntermediate.forward  s&    

=100?r&   r  r    r!   r"   r#   r   r@   r2   r   rG   r%   rP   rQ   s   @r'   r  r    s6    9z 9d 9U\\ ell  r&   r  c                   n   ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )	
BeitOutputi  rU   r+   Nc                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR                  5      U l	        g r>   )
r?   r@   r   r   r  rY   r   rh   ri   rj   r   s     r'   r@   BeitOutput.__init__  sB    YYv779K9KL
zz&"<"<=r&   rD   c                 J    U R                  U5      nU R                  U5      nU$ r>   r   rF   s     r'   rG   BeitOutput.forward  r   r&   r   r  rQ   s   @r'   r  r    s6    >z >d >
U\\ ell  r&   r  c                     ^  \ rS rSrSrSS\S\S-  S\SS4U 4S jjjr    SS	\	R                  S
\S\	R                  S-  S\S\\\4   S-  S\\	R                     \\	R                  \	R                  4   -  4S jjrSrU =r$ )	BeitLayeri  z?This corresponds to the Block class in the timm implementation.NrU   r   drop_path_rater+   c                   > [         TU ]  5         UR                  U l        SU l        [	        XS9U l        [        U5      U l        [        U5      U l	        [        R                  " UR                  UR                  S9U l        US:  a  [        U5      O[        R                   " 5       U l        [        R                  " UR                  UR                  S9U l        UR&                  nUS:  aw  [        R(                  " U[*        R,                  " UR                  5      -  SS9U l        [        R(                  " U[*        R,                  " UR                  5      -  SS9U l        g Su  U l        U l        g )	Nr   r   epsr-   r   T)requires_grad)NN)r?   r@   chunk_size_feed_forwardseq_len_dimr   r   r  intermediater  r8   r   	LayerNormrY   layer_norm_epslayernorm_beforer;   Identityr9   layernorm_afterlayer_scale_init_valuerW   r2   oneslambda_1lambda_2)rA   rU   r   r  init_valuesrB   s        r'   r@   BeitLayer.__init__  s   '-'E'E$&vG,V4 ( "V-?-?VEZEZ [9G#9Mn5SUS^S^S`!||F,>,>FDYDYZ33?LLuzz&BTBT7U)UeijDMLLuzz&BTBT7U)UeijDM+5(DM4=r&   rD   r   r   r   r   c                    U R                  U R                  U5      UUUUS9nUS   nUSS  nU R                  b  U R                  U-  nU R                  U5      U-   nU R	                  U5      n	U R                  U	5      n	U R                  U	5      n	U R                  b  U R                  U	-  n	U R                  U	5      U-   n	U	4U-   nU$ )Nr   r   r   r   r   r   )r   r"  r'  r9   r$  r  r8   r(  )
rA   rD   r   r   r   r   self_attention_outputsr  r   layer_outputs
             r'   rG   BeitLayer.forward  s     "&!!-0/#9%=! "0 "
 2!4(, ==$#}}/?? '78=H ++M:((6{{<0==$==<7L ~~l3mC/G+r&   )
r   r  r9   r  r'  r(  r$  r"  r8   r  )Nr-   r   )r    r!   r"   r#   r$   r   r   rN   r@   r2   r   r   r   rG   r%   rP   rQ   s   @r'   r  r    s    I6z 6 6]b 6mq 6 6* #(6:).-1'||'  ' !&t 3	'
 #'' #s(Od*' 
u||	uU\\5<<%?@	@' 'r&   r  c                      ^  \ rS rSrS\S\SS4U 4S jjr\" SS9S\\\4   S\	R                  4S	 j5       rSS
\S\	R                  4S jjrSrU =r$ )r   i  rU   r   r+   Nc                    > [         TU ]  5         X l        SUS   -  S-
  SUS   -  S-
  -  S-   U l        [        R
                  " [        R                  " U R                  UR                  5      5      U l	        g )Nrq   r   r   r   )
r?   r@   r   num_relative_distancer   rW   r2   rX   r   relative_position_bias_tabler   s      r'   r@   !BeitRelativePositionBias.__init__  sp    &&'+a.&81&<[QR^ASVWAW%X[\%\",.LLKK22F4N4NO-
)r&   
   )maxsizec                    SUS   -  S-
  SUS   -  S-
  -  S-   nUS   US   -  n[         R                  " [         R                  " US   5      [         R                  " US   5      SS9n[         R                  " U5      n[         R                  " US5      nUSS2SS2S4   USS2SSS24   -
  nUR                  SSS5      R                  5       nUSS2SS2S4==   US   S-
  -  ss'   USS2SS2S4==   US   S-
  -  ss'   USS2SS2S4==   SUS   -  S-
  -  ss'   [         R                  " US-   4S-  UR                  S9nUR                  S	5      USS2SS24'   US-
  USSS24'   US-
  USS2S4'   US-
  US
'   U$ )z
This method creates the relative position index, modified to support arbitrary window sizes,
as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
rq   r   r   r   ij)indexingN)rt   r.   rp   )r   r   )
r2   meshgridarangestackr   r|   r   rX   r.   sum)	rA   r   r2  window_areagridcoordscoords_flattenrelative_coordsrelative_position_indexs	            r'    generate_relative_position_index9BeitRelativePositionBias.generate_relative_position_index  s    "#[^!3a!7AA<NQR<R SVW W "!n{1~5~~ell;q>:ELLUV<XcghT"vq1(At4~aqj7QQ)11!Q:EEG1a KNQ$66 1a KNQ$66 1a AA$6$:: "'++K!O3E3IQ`QfQf"g*9*=*=b*AAB')>)B12&)>)BA&(=(A%&&r&   r   c                    SU R                   S   -  S-
  nSU R                   S   -  S-
  nSUS   -  S-
  nSUS   -  S-
  nU R                  nU R                  n	Xg-  S-   n
USU	S-
   nUR                  SXTS5      R	                  SSSS5      n[
        R                  R                  U[        U5      [        U5      4SS9nUR	                  SSSS5      R                  U
S-
  S5      n[        R                  " XU	S-
  S /5      nU R                  U5      nXR                  S5         nUR                  US   US   -  S-   US   US   -  S-   S5      nUR	                  SSS5      R                  5       nU(       a?  [
        R                  R                  UR                  S5      X34SS	S
9R                  S5      nUR                  S5      $ )ze
Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
rq   r   r   r   Nrp   bilinear)rt   ru   Frs   )r   r3  r2  r{   r|   r   r}   r~   r   r2   r   rD  r   r   r   squeeze)rA   r   r   r   
old_height	old_widthr   r    old_relative_position_bias_tableold_num_relative_distancenew_num_relative_distanceold_sub_tablenew_sub_table new_relative_position_bias_tablerC  r   s                   r'   rG    BeitRelativePositionBias.forward  s!    ))!,,q0
((++a/	Q'!+
A&*	+/+L+L($($>$>!$.$:Q$>!89X;TWX;XY%--aKSSTUWXZ[]^_11:!6	)8L MT^ 2 
 &--aAq9AAB[^_B_acd+099=VYZ=Z=\]^,
( #'"G"G"T!AB^B^_aBb!c "8!<!<N[^+a/Q+a.1PST1TVX"
 "8!?!?1a!H!S!S!U#%']]%>%>&003)#	 &? &
 gaj # &//22r&   )r2  r3  r   )FN)r    r!   r"   r#   r   r   r@   r   r   r2   r   rD  r   rG   r%   rP   rQ   s   @r'   r   r     sr    
z 
 
$ 
 )4'E#s(O 'PUP\P\ ' 5'0-3T -3]b]i]i -3 -3r&   r   c                      ^  \ rS rSrSS\S\S-  SS4U 4S jjjr     SS\R                  S\	S	\	S
\	S\\
\
4   S-  S\	S\\-  4S jjrSrU =r$ )BeitEncoderi<  NrU   r   r+   c                   > [         TU ]  5         Xl        UR                  U l        U R                  (       a  [        XS9U l        [        R                  " SUR                  UR                  SS9 Vs/ s H  o3R                  5       PM     nn[        R                  " [        UR                  5       Vs/ s H#  n[        UUR                   (       a  UOS XE   S9PM%     sn5      U l        SU l        g s  snf s  snf )Nr   r   cpu)r/   )r   r  F)r?   r@   rU   !use_shared_relative_position_biasr   r   r   r2   linspacer  num_hidden_layersitemr   
ModuleListranger  use_relative_position_biaslayergradient_checkpointing)rA   rU   r   xdprirB   s         r'   r@   BeitEncoder.__init__=  s    *0*R*R'***B6*cD' "'63H3H&JbJbkp!qr!qAvvx!qr]] v778 9A /5/P/PVZ#&6
 9	

 ',# ss   3C35*C8rD   r   output_hidden_statesr   r   return_dictc           	         U(       a  SOS nU(       a  SOS n[        U R                  5       H  u  pU(       a  Xq4-   nU R                  (       aR  Uu  pXR                  R                  -  XR                  R                  -  4nU R                  XUR                  S   S9nOS nU
" UUUUUS9nUS   nU(       d  M  XS   4-   nM     U(       a  Xq4-   nU(       d  [        S XU4 5       5      $ [        UUUS9$ )Nr   r   )r   r   r,  r   c              3   .   #    U  H  oc  M  Uv   M     g 7fr>   r   ).0vs     r'   	<genexpr>&BeitEncoder.forward.<locals>.<genexpr>|  s     m$[q$[s   	)last_hidden_staterD   
attentions)		enumerater]  r   rU   r_   r   r0   r   r   )rA   rD   r   rc  r   r   rd  all_hidden_statesall_self_attentionsra  layer_modulerm   rn   r   r   layer_outputss                   r'   rG   BeitEncoder.forwardR  s    #7BD$5b4(4OA#$58H$H!.. *%)?)??++J`J`A`a)-)D)D]j]p]pqr]s *E *& *.&("3'=)A%M *!,M  &91=M<O&O#1  54   14D Dm]GZ$[mmm++*
 	
r&   )rU   r^  r   r]  r   r>   )FFFNT)r    r!   r"   r#   r   r   r@   r2   r   r   r   r   rG   r%   rP   rQ   s   @r'   rS  rS  <  s    ,z , ,PT , ,0 #(%*).-1 /
||/
  /
 #	/

 #'/
 #s(Od*/
 /
 
	 /
 /
r&   rS  c                   |   ^  \ rS rSr% \\S'   SrSrSrSr	S/r
S/rSr\R                  " 5       U 4S	 j5       rS
rU =r$ )BeitPreTrainedModeli  rU   beit)imager   Tr  z.*relative_position_index.*c                   > [         TU ]  U5        [        U[        5      (       a|  [        R
                  " UR                  5        UR                  b   [        R
                  " UR                  5        UR                  b!  [        R
                  " UR                  5        gg[        U[        5      (       a!  [        R
                  " UR                  5        g[        U[        5      (       ay  UR                  bk  [        R                  " UR                  U R                  R                  5        [        R                  " UR                   U R                  R                  5        ggg)zInitialize the weightsN)r?   _init_weightsr`   rS   initzeros_rZ   r\   rg   r   r3  r  r'  	constant_rU   r%  r(  )rA   modulerB   s     r'   rx  !BeitPreTrainedModel._init_weights  s     	f%fn--KK(()  ,F--.))5F667 6 899KK;;<	***v0R0RSv0R0RS + +r&   r   )r    r!   r"   r#   r   __annotations__base_model_prefixinput_modalitiesmain_input_namesupports_gradient_checkpointing_no_split_modules"_keys_to_ignore_on_load_unexpected_supports_sdpar2   no_gradrx  r%   rP   rQ   s   @r'   rt  rt    sP    !$O&*#$*H)I&N
]]_T Tr&   rt  c                      ^  \ rS rSrSS\S\SS4U 4S jjjrS r\     SS\	R                  S	\	R                  S-  S
\S-  S\S-  S\S\S-  S\\-  4S jj5       rSrU =r$ )	BeitModeli  rU   add_pooling_layerr+   Nc                   > [         TU ]  U5        Xl        [        U5      U l        [        XR                  R                  R                  S9U l        UR                  (       a  [        R                  " 5       O([        R                  " UR                  UR                  S9U l        U(       a  [!        U5      OSU l        U R%                  5         g)z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
r   r  N)r?   r@   rU   rS   rl   rS  r^   r   encoderuse_mean_poolingr   r#  r   rY   r!  	layernorm
BeitPoolerpooler	post_init)rA   rU   r  rB   s      r'   r@   BeitModel.__init__  s    
 	 (0"67W7W7c7cd $44BKKM",,vGYGY_e_t_t:u 	 ->j(4 	r&   c                 .    U R                   R                  $ r>   rl   r^   rK   s    r'   get_input_embeddingsBeitModel.get_input_embeddings      ///r&   r   r   r   rc  r   rd  c           	         Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nU R	                  XS9u  pUR
                  SS n
U R                  UUUU
UUS9nUS   nU R                  U5      nU R                  b  U R                  U5      OSnU(       d  Ub  X4OU4nXSS -   $ [        UUUR                  UR                  S9$ )z
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
N)r   rq   )r   rc  r   rd  r   r   r   )rk  pooler_outputrD   rl  )rU   r   rc  rd  rl   r0   r  r  r  r   rD   rl  )rA   r   r   r   rc  r   rd  kwargsembedding_outputr   r   encoder_outputssequence_outputpooled_outputhead_outputss                  r'   rG   BeitModel.forward  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY"oolo\!''+
,,/!5!#%= ' 
 *!,..98<8OO4UY?L?XO;_n^pL!""555)-')77&11	
 	
r&   )rU   rl   r  r  r  )T)NNNFN)r    r!   r"   r#   r   r   r@   r  r   r2   r   r   r   r   rG   r%   rP   rQ   s   @r'   r  r    s    z d d  &0  48)-,0).#',
ll,
 ))D0,
  $;	,

 #Tk,
 #',
 D[,
 
+	+,
 ,
r&   r  c                   n   ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )	r  i  rU   r+   Nc                    > [         TU ]  5         UR                  (       a/  [        R                  " UR
                  UR                  S9U l        g S U l        g )Nr  )r?   r@   r  r   r   rY   r!  r  r   s     r'   r@   BeitPooler.__init__  sA    KQKbKbBLL++1F1FG 	hl 	r&   rD   c                     U R                   b0  US S 2SS 2S S 24   nU R                  UR                  S5      5      nU$ US S 2S4   nU$ )Nr   r   )r  mean)rA   rD   patch_tokensr  s       r'   rG   BeitPooler.forward  sU    >>%(AB2L NN<+<+<Q+?@M
  *!Q$/Mr&   )r  r  rQ   s   @r'   r  r    s6    
z 
d 
	U\\ 	ell 	 	r&   r  a  
    Beit Model transformer with a 'language' modeling head on top. BEiT does masked image modeling by predicting
    visual tokens of a Vector-Quantize Variational Autoencoder (VQ-VAE), whereas other vision models like ViT and DeiT
    predict RGB pixel values. As a result, this class is incompatible with [`AutoModelForMaskedImageModeling`], so you
    will need to use [`BeitForMaskedImageModeling`] directly if you wish to do masked image modeling with BEiT.
    c                      ^  \ rS rSrS\SS4U 4S jjrS r\       SS\R                  S-  S\R                  S-  S	\R                  S-  S
\S-  S\S-  S\S\S-  S\\-  4S jj5       rSrU =r$ )BeitForMaskedImageModelingi  rU   r+   Nc                 @  > [         TU ]  U5        UR                  U l        [        USS9U l        [
        R                  " UR                  UR                  S9U l	        [
        R                  " UR                  UR                  5      U l        U R                  5         g )NFr  r  )r?   r@   
num_labelsr  ru  r   r   rY   r!  r  r   
vocab_sizelm_headr  r   s     r'   r@   #BeitForMaskedImageModeling.__init__  su      ++f>	 f&8&8f>S>STyy!3!3V5F5FG 	r&   c                     g r>   r   rK   s    r'   get_output_embeddings0BeitForMaskedImageModeling.get_output_embeddings  s    r&   r   r   labelsr   rc  r   rd  c           	      j   Ub  UOU R                   R                  nU R                  UUUUUUS9n	U	S   n
U R                  U
5      n
U R	                  U
SS2SS24   5      nSnUb  [        5       nU" X   U5      nU(       d  U4U	SS -   nUb  U4U-   $ U$ [        UUU	R                  U	R                  S9$ )a  
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

Examples:

```python
>>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling
>>> import torch
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k")
>>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k")

>>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
>>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
>>> # create random boolean mask of shape (batch_size, num_patches)
>>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

>>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
>>> loss, logits = outputs.loss, outputs.logits
>>> list(logits.shape)
[1, 196, 8192]
```N)r   r   rc  r   rd  r   r   losslogitsrD   rl  )	rU   rd  ru  r  r  r   r   rD   rl  )rA   r   r   r  r   rc  r   rd  r  r   r  prediction_scoresmasked_lm_lossloss_fctr8   s                  r'   rG   "BeitForMaskedImageModeling.forward  s    \ &1%<k$++BYBY))+/!5%=#  
 "!*..9 LLAB)?@')H%&7&H&QN')GABK7F3A3M^%.YSYY$!//))	
 	
r&   )ru  r  r  r  )NNNNNFN)r    r!   r"   r#   r   r@   r  r   r2   r   r   r   r   r   rG   r%   rP   rQ   s   @r'   r  r    s    z d   -137&*)-,0).#'J
llT)J
 ))D0J
 t#	J

  $;J
 #TkJ
 #'J
 D[J
 
	J
 J
r&   r  z
    Beit Model transformer with an image classification head on top (a linear layer on top of the average of the final
    hidden states of the patch tokens) e.g. for ImageNet.
    c                      ^  \ rS rSrS\SS4U 4S jjr\      SS\R                  S-  S\R                  S-  S\	S-  S	\	S-  S
\	S\	S-  S\
\-  4S jj5       rSrU =r$ )BeitForImageClassificationic  rU   r+   Nc                 .  > [         TU ]  U5        UR                  U l        [        USS9U l        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l	        U R                  5         g )NTr  r   )r?   r@   r  r  ru  r   r   rY   r#  
classifierr  r   s     r'   r@   #BeitForImageClassification.__init__j  ss      ++f=	 OUN_N_bcNc"))F$6$68I8IJikititiv 	r&   r   r  r   rc  r   rd  c                 h   Ub  UOU R                   R                  nU R                  UUUUUS9nU(       a  UR                  OUS   n	U R	                  U	5      n
SnUb  U R                  X*U R                   5      nU(       d  U
4USS -   nUb  U4U-   $ U$ [        UU
UR                  UR                  S9$ )ab  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nr   rc  r   rd  r   rq   r  )	rU   rd  ru  r  r  loss_functionr   rD   rl  )rA   r   r  r   rc  r   rd  r  r   r  r  r  r8   s                r'   rG   "BeitForImageClassification.forwardv  s    " &1%<k$++BYBY))/!5%=#  
 2=--'!*/%%fdkkBDY,F)-)9TGf$EvE$!//))	
 	
r&   )ru  r  r  NNNNFN)r    r!   r"   r#   r   r@   r   r2   r   r   r   r   rG   r%   rP   rQ   s   @r'   r  r  c  s    
z 
d 
  -1&*)-,0).#'*
llT)*
 t#*
  $;	*

 #Tk*
 #'*
 D[*
 
&	&*
 *
r&   r  c                      ^  \ rS rSrSr   SS\S\S\\\\4   -  S\\\\4   -  \-  S\S\\\\4   -  S	S
4U 4S jjjr	S\
R                  S	\
R                  4S jrSrU =r$ )BeitConvModulei  a4  
A convolutional block that bundles conv/norm/activation layers. This block simplifies the usage of convolution
layers, which are commonly used with a norm layer (e.g., BatchNorm) and activation layer (e.g., ReLU).

Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
in_channelsout_channelsr   paddingr   dilationr+   Nc           	         > [         TU ]  5         [        R                  " UUUUUUS9U l        [        R
                  " U5      U l        [        R                  " 5       U l        g )N)r  r  r   r  r   r  )	r?   r@   r   r   convBatchNorm2dbnReLU
activation)rA   r  r  r   r  r   r  rB   s          r'   r@   BeitConvModule.__init__  sQ     	II#%#
	 ...'')r&   r(   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ r>   )r  r  r  )rA   r(   r8   s      r'   rG   BeitConvModule.forward  s0    5!(r&   )r  r  r  )r   Fr   )r    r!   r"   r#   r$   r   r   rO   r   r@   r2   r   rG   r%   rP   rQ   s   @r'   r  r    s     01*+$$ $ 5c?*	$
 uS#X&,$ $ c3h'$ 
$ $*U\\ ell  r&   r  c                   v   ^  \ rS rSrS\S\S\SS4U 4S jjrS\R                  S\R                  4S	 jrS
r	U =r
$ )BeitPyramidPoolingBlocki  
pool_scaler  channelsr+   Nc                    > [         TU ]  5         [        R                  " U5      [	        X#SS9/U l        [        U R
                  5       H   u  pEU R                  [        U5      U5        M"     g )Nr   r   )	r?   r@   r   AdaptiveAvgPool2dr  layersrm  
add_modulerO   )rA   r  r  r  ra  r]  rB   s         r'   r@    BeitPyramidPoolingBlock.__init__  sX      ,;a@
 "$++.HAOOCFE* /r&   r(   c                 @    UnU R                    H  nU" U5      nM     U$ r>   r  )rA   r(   hidden_stater]  s       r'   rG   BeitPyramidPoolingBlock.forward  s%    [[E .L !r&   r  )r    r!   r"   r#   r   r@   r2   r   rG   r%   rP   rQ   s   @r'   r  r    sD    +3 +S +C +D +U\\ ell  r&   r  c            
          ^  \ rS rSrSrS\\S4   S\S\S\SS	4
U 4S
 jjrS\	R                  S\\	R                     4S jrSrU =r$ )BeitPyramidPoolingModulei  a  
Pyramid Pooling Module (PPM) used in PSPNet.

Args:
    pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
        Module.
    in_channels (int): Input channels.
    channels (int): Channels after modules, before conv_seg.
    align_corners (bool): align_corners argument of F.interpolate.

Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
pool_scales.r  r  rv   r+   Nc                   > [         TU ]  5         Xl        X@l        X l        X0l        / U l        [        U5       HE  u  pV[        XbUS9nU R                  R                  U5        U R                  [        U5      U5        MG     g )N)r  r  r  )r?   r@   r  rv   r  r  blocksrm  r  appendr  rO   )	rA   r  r  r  rv   ra  r  blockrB   s	           r'   r@   !BeitPyramidPoolingModule.__init__  sn    &*& &{3MA+zemnEKKu%OOCFE* 4r&   r_  c                     / nU R                    HV  nU" U5      n[        R                  R                  XAR	                  5       SS  SU R
                  S9nUR                  U5        MX     U$ )Nrq   rG  rs   )r  r   r}   r~   rt   rv   r  )rA   r_  ppm_outsppmppm_outupsampled_ppm_outs         r'   rG    BeitPyramidPoolingModule.forward  sg    ;;C!fG " 9 9ffhqrl4K]K] !: ! OO-.  r&   )rv   r  r  r  r  )r    r!   r"   r#   r$   r   r   r   r@   r2   r   listrG   r%   rP   rQ   s   @r'   r  r    s`    
+E#s(O 
+# 
+QT 
+ei 
+nr 
+ $u||*<  r&   r  c                   x   ^  \ rS rSrSrS\SS4U 4S jjrS rS\R                  S\R                  4S	 jr
S
rU =r$ )BeitUperHeadi  z
Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
[UPerNet](https://huggingface.co/papers/1807.10221).

Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
rU   r+   Nc                   > [         TU ]  5         UR                  U l        UR                  /S-  U l        UR                  U l        SU l        [        R                  " U R
                  UR                  SS9U l
        [        U R                  U R                  S   U R
                  U R                  S9U l        [        U R                  S   [        U R                  5      U R
                  -  -   U R
                  SSS9U l        [        R                   " 5       U l        [        R                   " 5       U l        U R                  S S  Hm  n[        X R
                  SS9n[        U R
                  U R
                  SSS9nU R"                  R'                  U5        U R$                  R'                  U5        Mo     [        [        U R                  5      U R
                  -  U R
                  SSS9U l        g )	N   Fr   r  rp   )rv   r   r   r  )r?   r@   r  rY   r  r  rv   r   r   r  r  r  psp_modulesr  len
bottleneckrZ  lateral_convs	fpn_convsr  fpn_bottleneck)rA   rU   r  l_convfpn_convrB   s        r'   r@   BeitUperHead.__init__  s   !--"../!3**"))DMM63D3DRST 4R MM,,	
 )R 3t'7'7#84==#HHMM	
  ]]_++CR0K#KANF%dmmT]]PQ[\]H%%f-NN!!(+	 1 -  !DMM1MM	
r&   c                     US   nU/nUR                  U R                  U5      5        [        R                  " USS9nU R	                  U5      nU$ r   )extendr  r2   r   r  )rA   inputsr_  psp_outsr8   s        r'   psp_forwardBeitUperHead.psp_forward-  sL    2J3((+,99X1-*r&   encoder_hidden_statesc           	      @   [        U R                  5       VVs/ s H  u  p#U" X   5      PM     nnnUR                  U R                  U5      5        [	        U5      n[        US-
  SS5       HP  nXBS-
     R                  SS  nXBS-
     [        R                  R                  XB   USU R                  S9-   XBS-
  '   MR     [        US-
  5       Vs/ s H  o R                  U   " XB   5      PM     nnUR                  US   5        [        US-
  SS5       HA  n[        R                  R                  Xr   US   R                  SS  SU R                  S9Xr'   MC     [        R                  " USS9nU R                  U5      nU R                  U5      nU$ s  snnf s  snf )Nr   r   rp   rq   rG  rs   rw   )rm  r  r  r  r  r[  r0   r   r}   r~   rv   r  r2   r   r  r  )	rA   r  ra  lateral_convlateralsused_backbone_levels
prev_shapefpn_outsr8   s	            r'   rG   BeitUperHead.forward6  s   R[\`\n\nRopRoqL!6!9:Rop(()>?@  #8}+a/B7A!a%..qr2J&1uo0I0I*:TM_M_ 1J 1 HUO 8 =BBVYZBZ<[\<[qNN1%hk2<[\%+a/B7A--33(1+"3"3AB"7jX\XjXj 4 HK 8 99X1-$$X.(3 q ]s   F F)
rv   r  r  r  r  r  r  r  r  r  )r    r!   r"   r#   r$   r   r@   r  r2   r   rG   r%   rP   rQ   s   @r'   r  r    sA    $
z $
d $
LU\\ ell  r&   r  c                      ^  \ rS rSrSr SS\S\S\S\\\\4   -  SS4
U 4S	 jjjrS
\	R                  S\	R                  4S jrSrU =r$ )BeitFCNHeadiT  a  
Fully Convolution Networks for Semantic Segmentation. This head is implemented of
[FCNNet](https://huggingface.co/papers/1411.4038>).

Args:
    config (BeitConfig): Configuration.
    in_channels
    kernel_size (int): The kernel size for convs in the head. Default: 3.
    dilation (int): The dilation rate for convs in the head. Default: 1.


Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
rU   in_indexr   r  r+   Nc                 2  > [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        UR                  U l	        X l
        US-  U-  n/ nUR                  [        U R                  U R
                  X5US95        [        U R                  S-
  5       H2  nUR                  [        U R
                  U R
                  X5US95        M4     U R                  S:X  a  [        R                  " 5       U l        O[        R"                  " U6 U l        U R                  (       a4  [        U R                  U R
                  -   U R
                  X3S-  S9U l        [        R&                  " U R
                  UR(                  SS9U l        g )Nrq   )r   r  r  r   r   r  r  )r?   r@   rY   r  auxiliary_channelsr  auxiliary_num_convs	num_convsauxiliary_concat_inputconcat_inputr  r  r  r[  r   r#  convs
Sequentialconv_catr   r  r  )	rA   rU   r  r   r  conv_paddingr  ra  rB   s	           r'   r@   BeitFCNHead.__init__c  sH    	!--1133"99 #q(H4  $--[iq	

 t~~)*ALLMM4==kjr + >>QDJ.DJ*  4==0$--[qrbrDM ))DMM63D3DRSTr&   r  c                     XR                      nU R                  U5      nU R                  (       a%  U R                  [        R
                  " X#/SS95      nU R                  U5      nU$ )Nr   rw   )r  r  r  r  r2   r   r  )rA   r  rD   r8   s       r'   rG   BeitFCNHead.forward  sT    -mm<M*]]599m-D!#LMF(r&   )r  r  r  r  r  r  r  r  )rq   r   r   )r    r!   r"   r#   r$   r   r   r   r@   r2   r   rG   r%   rP   rQ   s   @r'   r  r  T  s{     no U  U,/ UBE UUX[`adfiai[jUj U	 U  UDU\\ ell  r&   r  c                      ^  \ rS rSrS\SS4U 4S jjrS r\      SS\R                  S-  S\R                  S-  S	\
S-  S
\
S-  S\
S\
S-  S\\-  4S jj5       rSrU =r$ )BeitForSemanticSegmentationi  rU   r+   Nc                 x  > [         TU ]  U5        UR                  U l        [        USS9U l        [        U R                  R                  5      S:w  a  [        S5      e[        R                  " [        R                  " UR                  UR                  SSS9[        R                  " UR                  5      [        R                  " 5       [        R                  " UR                  UR                  SSS95      U l        [        R                  " [        R                  " UR                  UR                  SSS95      U l        [        R"                  " 5       U l        [        R&                  " SSS9U l        [+        U5      U l        UR.                  (       a  [1        U5      OS U l        U R5                  5         g )NFr  r  zBeitForSemanticSegmentation requires config.out_indices to be a list of 4 integers, specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of a base-sized architecture.rq   r   )r?   r@   r  r  ru  r  rU   out_indicesr   r   r  ConvTranspose2drY   r  GELUfpn1fpn2r#  fpn3	MaxPool2dfpn4r  decode_headuse_auxiliary_headr  auxiliary_headr  r   s     r'   r@   $BeitForSemanticSegmentation.__init__  sO     ++f>	 t{{&&'1,- 
 MMv1163E3EST]^_NN6--.GGIv1163E3EST]^_	
	 MMv1163E3EST]^_
	 KKM	LLQq9	 (/5;5N5Nk&1TX 	r&   c                 X   [         R                  R                  XR                  SS  SSS9nUb,  [         R                  R                  X#R                  SS  SSS9n[	        U R
                  R                  S9nU" XC5      nUnUb$  U" WU5      n	XR
                  R                  U	-  -  nU$ )Nr   rG  Frs   )ignore_index)r   r}   r~   r0   r   rU   semantic_loss_ignore_indexauxiliary_loss_weight)
rA   r  auxiliary_logitsr  upsampled_logitsupsampled_auxiliary_logitsr  	main_lossr  auxiliary_losss
             r'   compute_loss(BeitForSemanticSegmentation.compute_loss  s    ==44bc*5 5 
 ')+)B)B ||BC'8zY^ *C *& $1W1WX-6	'%&@&INKK55FFDr&   r   r  r   rc  r   rd  c           	         Ub  UOU R                   R                  nUb  UOU R                   R                  nUb%  U R                   R                  S:X  a  [	        S5      eU R                  UUSUUS9nU(       a  UR                  OUS   n	[        U	5       V
Vs/ s H&  u  pU
S-   U R                   R                  ;   d  M$  UPM(     nn
nUR                  S   nU R                   R                  U R                   R                  -  nU Vs/ s H2  oSS2SS2SS24   R                  SSS5      R                  USX5      PM4     nnU R                  U R                  U R                   U R"                  /n[%        ['        U5      5       H  nUU   " UU   5      UU'   M     U R)                  U5      nSnU R*                  b  U R+                  U5      nSnUb  U R-                  UUU5      nU(       d%  U(       a
  U4USS -   nO	U4USS -   nUb  U4U-   $ U$ [/        UUU(       a  UR                  OSUR0                  S	9$ s  snn
f s  snf )
a  
labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
    Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

Examples:

```python
>>> from transformers import AutoImageProcessor, BeitForSemanticSegmentation
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")
>>> model = BeitForSemanticSegmentation.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")

>>> inputs = image_processor(images=image, return_tensors="pt")
>>> outputs = model(**inputs)
>>> # logits are of shape (batch_size, num_labels, height, width)
>>> logits = outputs.logits
```Nr   z/The number of labels should be greater than oneTr  r   rq   rp   r  )rU   rd  rc  r  r   ru  rD   rm  r   r0   ra   r_   r|   r{   r#  r$  r%  r'  r[  r  r(  r*  r5  r   rl  )rA   r   r  r   rc  r   rd  r  r   r  idxfeaturefeaturesr   patch_resolutionr_  opsra  r  r0  r  r8   s                         r'   rG   #BeitForSemanticSegmentation.forward  sX   H &1%<k$++BYBY$8$D $++JjJj 	 $++"8"8A"=NOO))/!%%=#  
 :E 5 5'RS* 1::O0Pw0PTWZ[T[_c_j_j_v_vTvG0Pw!''*
;;11T[[5K5KKnv
nvijaQhK1a(00RAQdnv 	 

 yy$))TYY		:s8}%Aa&!-HQK & !!(+*#228<$$V-=vFD# WQR[0 WQR[0)-)9TGf$EvE&3G'//T))	
 	
; x
s   #H;?H;9I)r*  ru  r(  r#  r$  r%  r'  r  r  )r    r!   r"   r#   r   r@   r5  r   r2   r   r   r   r   rG   r%   rP   rQ   s   @r'   r  r    s    z d @&  -1&*)-,0).#'Y
llT)Y
 t#Y
  $;	Y

 #TkY
 #'Y
 D[Y
 
(	(Y
 Y
r&   r  zM
    BEiT backbone, to be used with frameworks like DETR and MaskFormer.
    c                      ^  \ rS rSrU 4S jrS r\\\   SS\	S\
S-  S\
S-  S\
S-  S	\4
S
 jj5       5       5       rSrU =r$ )BeitBackbonei!  c                 |  > [         TU ]  U5        [        UR                  S-   5       Vs/ s H  o!R                  PM     snU l        [        U5      U l        [        XR                  R                  R                  S9U l        UR                  (       Ga  [        U R                  R                  5      S:w  a  [!        S5      eUR                  n["        R$                  " ["        R&                  " X3SSS9["        R(                  " X1R*                  S9["        R,                  " 5       ["        R&                  " X3SSS95      U l        ["        R$                  " ["        R&                  " X3SSS95      U l        ["        R2                  " 5       U l        ["        R6                  " SSS9U l        U R;                  5         g s  snf )Nr   r   r  zBeitBackbone requires config.out_indices to be a list of 4 integers, specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of a base-sized architecture.rq   r   r  )r?   r@   r[  rX  rY   num_featuresrS   rl   rS  r^   r   r  add_fpnr  rU   r   r   r   r  r!  r  batch_norm_epsr"  r#  r$  r#  r%  r&  r'  r  )rA   rU   r   rY   rB   s       r'   r@   BeitBackbone.__init__'  sJ    9>v?W?WZ[?[9\]9\A//9\](0"67W7W7c7cd>>>4;;**+q0 1 
 !,,K"";STU{0E0EF	"";STU	DI b&8&8_`ij&klDIDI1=DI 	1 ^s   F9c                 .    U R                   R                  $ r>   r  rK   s    r'   r  !BeitBackbone.get_input_embeddingsD  r  r&   Nr   rc  r   rd  r+   c                    Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUR                  S   nU R                  U5      u  nu  pUR                  SS n
U R                  USUU
US9nU(       a  UR                  OUS   nSn[        U R                  U5       Hj  u  pXR                  ;   d  M  U R                   R                  (       a4  USS2SS2SS24   nUR                  SSS5      nUR                  USX5      nX4-  nMl     U R                   R                  (       aY  U R                  US   5      U R!                  US   5      U R#                  US   5      U R%                  US	   5      /n['        U5      nU(       d  U(       a  U4USS -   nU$ U4USS -   nU$ [)        UU(       a  UR                  OSUR*                  S
9$ )a  
Examples:

```python
>>> from transformers import AutoImageProcessor, AutoBackbone
>>> import torch
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224")
>>> model = AutoBackbone.from_pretrained(
...     "microsoft/beit-base-patch16-224", out_features=["stage1", "stage2", "stage3", "stage4"]
... )

>>> inputs = processor(image, return_tensors="pt")

>>> outputs = model(**inputs)
>>> feature_maps = outputs.feature_maps
>>> list(feature_maps[-1].shape)
[1, 768, 14, 14]
```Nr   rq   T)rc  r   r   rd  r   r   rp   r   )feature_mapsrD   rl  )rU   rd  rc  r   r0   rl   r  rD   zipstage_namesout_featuresreshape_hidden_statesr|   r{   rB  r#  r$  r%  r'  r   r   rl  )rA   r   rc  r   rd  r  r   r  r   r   r   r   rD   rH  stager  r8   s                    r'   rG   BeitBackbone.forwardG  s    J &1%<k$++BYBY$8$D $++JjJj 	 2C1N-TXT_T_TqTq!''*
8<8U55<!''+
,,!%/!#  
 2=--'!*#&t'7'7#GE)));;44#/12q#9L#/#7#71a#@L#/#7#7
B#bL/ $H ;;		,q/*		,q/*		,q/*		,q/*	L !.L#&712;6 M '712;6M%3G'//T))
 	
r&   )rl   r  r#  r$  r%  r'  rA  )NNN)r    r!   r"   r#   r@   r  r   r   r   r   r   r   rG   r%   rP   rQ   s   @r'   r?  r?  !  s    :0   -1)-#'T
T
 #TkT
  $;	T

 D[T
 
T
  ! T
r&   r?  )r  r  r  r  rt  r?  )r-   F)Jr$   collections.abcrb   r   dataclassesr   r2   r   r   torch.nnr    r   ry  activationsr	   backbone_utilsr
   r   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_utilsr   pytorch_utilsr   utilsr   r   r   utils.genericr   configuration_beitr   
get_loggerr    r   r   rN   r   r9   Moduler;   rS   r]   r   r   r   r   r   r  r  r  r   rS  rt  r  r  r  r  r  r  r  r  r  r  r?  __all__r   r&   r'   <module>r_     s      !   % & ! H 9  . @ 7 7 - * 
		H	% 
!;  U\\ e T V[VbVb %299 % \7RYY \7~#7")) #7LF		 FR0#- 0#fRYY & ! BII 0ryy  
 
<* <~P3ryy P3fE
")) E
P T/ T T8 D
# D
 D
N & \
!4 \
\
~ 8
!4 8
8
v"RYY "Jbii ""ryy "JR299 Rj8")) 8v N
"5 N
 N
b 
x
="5 x

x
vr&   