
    Z j|                        S r SSKrSSKrSSKJs  Jr  SSKJr  SSKJr  SSK	J
r  SSKJr  SSKJrJr  SS	KJr  SS
KJrJr  SSKJr  SSKJrJr  SSKJr  \R:                  " \5      r " S S\R@                  5      r! " S S\R@                  5      r" " S S\R@                  5      r# " S S\R@                  5      r$ " S S\R@                  5      r% " S S\R@                  5      r& " S S\R@                  5      r' " S S\R@                  5      r( " S S \R@                  5      r) " S! S"\R@                  5      r* " S# S$\R@                  5      r+\ " S% S&\5      5       r,\ " S' S(\,5      5       r-\" S)S*9 " S+ S,\,\5      5       r./ S-Qr/g).zPyTorch CPMAnt    N)nn)CrossEntropyLoss   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)auto_docstringlogging   )CpmAntConfigc                   V   ^  \ rS rSrSrS\4U 4S jjrS\R                  4S jr	Sr
U =r$ )CpmAntLayerNorm$   zv
We use Root Mean Square (RMS) Layer Normalization, please see https://huggingface.co/papers/1910.07467 for details."
configc                    > [         TU ]  5         UR                  U l        UR                  U l        [
        R                  " [        R                  " UR                  5      5      U l	        g N)
super__init__epshidden_sizedim_normr   	Parametertorchemptyweightselfr   	__class__s     {/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/cpmant/modeling_cpmant.pyr   CpmAntLayerNorm.__init__)   sE    ::**ll5;;v/A/A#BC    hidden_statesc                 l   UR                  S5      U R                  :w  a  [        S5      eUR                  nUR	                  [
        R                  5      R                  S5      R                  SSS9nU[
        R                  " X0R                  -   5      -  R	                  U5      U R                  -  nU$ )N
Args:
    hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`)
z'hidden_states.size(-1) != self.dim_norm   T)dimkeepdim)sizer   AssertionErrordtypetor   float32powmeanrsqrtr   r    )r"   r'   	old_dtypevariances       r$   forwardCpmAntLayerNorm.forward0   s    
 b!T]]2 !JKK!''	 ##EMM266q9>>2t>T&X5H)IIMMiX[_[f[ffr&   )r   r   r    )__name__
__module____qualname____firstlineno____doc__r   r   r   Tensorr8   __static_attributes____classcell__r#   s   @r$   r   r   $   s+    D| D
U\\ 
 
r&   r   c                      ^  \ rS rSrSS\4U 4S jjjr   SS\R                  S\R                  S\R                  S\R                  S	\	S-  S
\
S-  S\	S-  4S jjrSrU =r$ )CpmAntAttention=   Nr   c                 :  > [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        X l        [        R                  " U R                  U R
                  U R                  -  SS9U l
        [        R                  " U R                  U R
                  U R                  -  SS9U l        [        R                  " U R                  U R
                  U R                  -  SS9U l        [        R                  " U R
                  U R                  -  U R                  SS9U l        [        R                  R                  SS9U l        UR"                  b-  [        R                  R%                  UR"                  S9U l        g S U l        g )NFbiasr*   r,   )p)r   r   r   	dim_modelnum_attention_heads	num_headsdim_head	layer_idxr   Linear	project_q	project_k	project_vattention_outr   Softmaxsoftmax	dropout_pDropoutdropoutr"   r   rO   r#   s      r$   r   CpmAntAttention.__init__>   s   ++33"4>>4>>DMM3QX]^4>>4>>DMM3QX]^4>>4>>DMM3QX]^YYt~~'Et~~\abxx''B'/' 88++f.>.>+?DLDLr&   hidden_q	hidden_kvattention_maskposition_biasoutput_attentionspast_key_values	use_cachec           
      z   UR                  S5      n	UR                  S5      n
UR                  S5      nU R                  U5      nU R                  U5      nU R                  U5      nUR	                  XU R
                  U R                  5      R                  SSSS5      nUR	                  XU R
                  U R                  5      R                  SSSS5      nUR	                  XU R
                  U R                  5      R                  SSSS5      nUb/  UR                  XU R                  5      u  pUR                  S5      n[        R                  " XR                  SS5      5      [        R                  " U R                  5      -  nX-   n[        R                  " UUR	                  U	SX5      [        R                   " S5      :H  [        R"                  " [%        S	5      UR&                  UR(                  S
95      nU R+                  U5      n[        R                  " UUR	                  U	SX5      [        R                   " S5      :H  [        R"                  " SUR&                  UR(                  S
95      nU(       a  UnOSnU R,                  b  U R-                  U5      n[        R                  " X5      nUR	                  XR
                  XR                  5      R                  SSSS5      nUR/                  5       R	                  XU R
                  U R                  -  5      nU R1                  U5      nUU4$ )a  
Args:
    hidden_q (`torch.Tensor`):
        Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences.
    hidden_kv (`torch.Tensor` of shape `(batch, len_k, dim_model)`)):
        Tensor *key_value* and *query* of shape `(batch, len_k, dim_model)`
    attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
        Avoid invalid areas to participate in the calculation of self-attention.
    position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
        Provide positional information to self-attention block.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers.
    past_key_values (`Cache`, *optional*):
        Cached past key and value projection states.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
r   r   r+   r   Nr*   Fz-inf)devicer0   )r.   rQ   rR   rS   viewrM   rN   permuteupdaterO   r   matmul	transposemathsqrtmasked_filltensorscalar_tensorfloatre   r0   rV   rY   
contiguousrT   )r"   r\   r]   r^   r_   r`   ra   rb   kwargs
batch_sizelen_qlen_kquerykeyvaluescoreattn_weightss                    r$   r8   CpmAntAttention.forwardR   s   : ]]1%
a q!x(nnY'y)

:dnndmmLTTUVXY[\^_`hhz$..$--HPPQRTUWXZ[\

:dnndmmLTTUVXY[\^_`&(//DNNKJCHHRLE UMM"b$9:TYYt}}=UU%!!
Au<U@SSfell%++V

 U#!!
Au<U@SS%,,ekkJ

  LL<<#LL'E U*

:~~ummLTTUVXY[\^_`  "''
4>>DMM;YZ""5)l""r&   )
rT   rN   rK   rY   rO   rM   rR   rQ   rS   rV   r   )FNN)r:   r;   r<   r=   r   r   r   r?   
BoolTensorboolr   r8   r@   rA   rB   s   @r$   rD   rD   =   s     |    4 */(,!%M#,,M# <<M# ((	M#
 ||M#  $;M# M# $;M# M#r&   rD   c                      ^  \ rS rSrSS\4U 4S jjjr    SS\R                  S\R                  S\R                  S-  S\S-  S	\	S-  S
\S-  4S jjr
SrU =r$ )CpmAntSelfAttentionBlock   Nr   c                    > [         TU ]  5         [        U5      U l        [	        XS9U l        UR                  (       a/  [        R                  R                  UR                  5      U l
        g S U l
        g N)rO   )r   r   r   layernorm_before_attentionrD   self_attentionrW   r   r   rX   rY   rZ   s      r$   r   !CpmAntSelfAttentionBlock.__init__   sT    *9&*A'-fJ 88++F,<,<=DLDLr&   r'   r^   r_   r`   ra   rb   c           	          U R                  U5      nU R                  UUUUUUU5      u  pU R                  b  U R                  U5      nX-   nX4$ )av  
Args:
    hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`):
        Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences.
    attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
        Avoid invalid areas to participate in the calculation of self-attention.
    position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
        Provide positional information to self-attention block.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers.
    past_key_values (`Cache`, *optional*):
        Cached past key and value projection states.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
)r   r   rY   )
r"   r'   r^   r_   r`   ra   rb   rr   outputsrz   s
             r$   r8    CpmAntSelfAttentionBlock.forward   sg    4 11-@ $ 3 3!
 <<#ll7+G%/**r&   )rY   r   r   r   NFNNr:   r;   r<   r=   r   r   r   r?   r}   r   r8   r@   rA   rB   s   @r$   r   r      s     |     .2).(,!%)+||)+ )+ ||d*	)+
  $;)+ )+ $;)+ )+r&   r   c                   R   ^  \ rS rSrS\4U 4S jjrS\R                  4S jrSr	U =r
$ )CpmAntDenseGatedACT   r   c                 $  > [         TU ]  5         [        R                  " UR                  UR
                  SS9U l        [        R                  " UR                  UR
                  SS9U l        [        R                  R                  5       U l
        g NFrG   )r   r   r   rP   r   dim_ffw_0w_1r   GELUactr!   s     r$   r   CpmAntDenseGatedACT.__init__   s[    99V//UK99V//UK88==?r&   r'   c                 p    U R                  U R                  U5      5      nU R                  U5      nX!-  nU$ )zTransform an input tensor from one feature space to another via a nonlinear operation

Args:
    hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`)
r   r   r   )r"   r'   
gate_scores      r$   r8   CpmAntDenseGatedACT.forward   s7     XXdhh}56
/"2r&   r   r:   r;   r<   r=   r   r   r   r?   r8   r@   rA   rB   s   @r$   r   r      s$    #| #
U\\ 
 
r&   r   c                   R   ^  \ rS rSrS\4U 4S jjrS\R                  4S jrSr	U =r
$ )CpmAntFeedForward   r   c                 &  > [         TU ]  5         [        U5      U l        UR                  b/  [
        R                  R                  UR                  5      U l        OS U l        [        R                  " UR                  UR                  SS9U l        g r   )r   r   r   w_inrW   r   r   rX   rY   rP   r   r   w_outr!   s     r$   r   CpmAntFeedForward.__init__   sg    '/	' 88++F,<,<=DLDLYYv}}f.@.@uM
r&   r'   c                     U R                  U5      nU R                  b  U R                  U5      nU R                  U5      nU$ )r)   )r   rY   r   r"   r'   s     r$   r8   CpmAntFeedForward.forward   s>    
 		-0<<# LL7M

=1r&   )rY   r   r   r   rB   s   @r$   r   r      s&    N| NU\\  r&   r   c                   R   ^  \ rS rSrS\4U 4S jjrS\R                  4S jrSr	U =r
$ )CpmAntFFNBlocki  r   c                    > [         TU ]  5         [        U5      U l        [	        U5      U l        UR                  (       a/  [        R                  R                  UR                  5      U l
        g S U l
        g r   )r   r   r   layernorm_before_ffnr   ffnrW   r   r   rX   rY   r!   s     r$   r   CpmAntFFNBlock.__init__  sS    $3F$;!$V, 88++F,<,<=DLDLr&   r'   c                     U R                  U5      nU R                  U5      nU R                  b  U R                  U5      nX-   nU$ )z
Args:
    hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`):
        Hidden states before feed forward layer.
)r   r   rY   )r"   r'   
ln_outputsr   s       r$   r8   CpmAntFFNBlock.forward  sH     ..}=
((:&<<#ll7+G%/r&   )rY   r   r   r   rB   s   @r$   r   r     s%     |  || r&   r   c                      ^  \ rS rSrSS\4U 4S jjjr    SS\R                  S\R                  S\R                  S-  S\S-  S	\	S-  S
\S-  4S jjr
SrU =r$ )CpmAntTransformerBlocki!  Nr   c                 ^   > [         TU ]  5         [        XS9U l        [	        U5      U l        g r   )r   r   r   self_attr   r   rZ   s      r$   r   CpmAntTransformerBlock.__init__"  s&    0M!&)r&   r'   r^   r_   r`   ra   rb   c           	      V    U R                  UUUUUUS9u  pU R                  U5      nX4$ )a  
Args:
    hidden_states (`torch.Tensor`):
        Input to the layer of shape `(batch, seq_len, dim_model)`
    attention_mask (`torch.Tensor`):
        Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)`
    position_bias (`torch.Tensor`):
        Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)`
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers.
    past_key_values (`Cache`, *optional*):
        Cached past key and value projection states
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
)r^   r_   r`   ra   rb   )r   r   )	r"   r'   r^   r_   r`   ra   rb   rr   rz   s	            r$   r8   CpmAntTransformerBlock.forward'  sE    4 '+mm)'/+ '4 '
# /**r&   )r   r   r   r   r   rB   s   @r$   r   r   !  s    *| * * .2).(,!%$+||$+ $+ ||d*	$+
  $;$+ $+ $;$+ $+r&   r   c                      ^  \ rS rSrS\4U 4S jjr    SS\R                  S\R                  S\R                  S\S-  S	\S-  S
\	S-  S\S-  4S jjr
SrU =r$ )CpmAntEncoderiN  r   c           
         > [         TU ]  5         UR                  U l        [        R
                  " [        U R                  5       Vs/ s H  n[        XS9PM     sn5      U l        [        U5      U l
        g s  snf r   )r   r   num_hidden_layers
num_layersr   
ModuleListranger   layersr   output_layernorm)r"   r   ir#   s      r$   r   CpmAntEncoder.__init__O  sc     22mmZ_`d`o`oZp$qZpUV%;F%PZp$qr / 7 %rs   A8Nr'   r^   r_   r`   output_hidden_statesra   rb   c           
          U(       a  SOSn	U(       a  SOSn
[        U R                  5       H.  u  pU(       a  X4-  n	U" UUUUUUS9nUu  pU(       d  M)  X4-  n
M0     U R                  U5      nU(       a  X4-  n	XU
4$ )a  
Args:
    hidden_states (`torch.Tensor`):
        Input to the layer of shape `(batch, seq_len, dim_model)`
    attention_mask (`torch.Tensor`):
        Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)`
    position_bias (`torch.Tensor`):
        Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)`
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers.
    output_hidden_states (`bool`, *optional*):
        Whether or not to return the hidden states of all layers.
    past_key_values (`Cache`, *optional*):
        Cached past key and value projection states
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
 N)r`   ra   rb   )	enumerater   r   )r"   r'   r^   r_   r`   r   ra   rb   rr   all_hidden_statesall_self_attnsr   layerlayer_outputsrz   s                  r$   r8   CpmAntEncoder.forwardV  s    : #7BD0d!$++.HA#!%55!!"3 /#M +8'M  /1 / --m<!11??r&   )r   r   r   )NNNNr   rB   s   @r$   r   r   N  s    8| 8 *.,0(,!%4@||4@ 4@ ||	4@
  $;4@ #Tk4@ 4@ $;4@ 4@r&   r   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )CpmAntIntermediatei  c                   > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g r   )r   r   r   rP   r   intermediate_sizedense
isinstance
hidden_actstrr   intermediate_act_fnr!   s     r$   r   CpmAntIntermediate.__init__  s`    YYv1163K3KL
f''--'-f.?.?'@D$'-'8'8D$r&   r'   returnc                 J    U R                  U5      nU R                  U5      nU$ r   r   r   r   s     r$   r8   CpmAntIntermediate.forward  s&    

=100?r&   r   
r:   r;   r<   r=   r   r   r?   r8   r@   rA   rB   s   @r$   r   r     s(    9U\\ ell  r&   r   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  S\R                  4S jrS	 r	SS
 jr
SrU =r$ )CpmAntSegmentPositionEmbeddingi  r   c                 f  > [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        UR                  U l	        [        R                  " [        R                  " UR                  UR                  -  UR                  -   UR                  5      5      U l        g r   )r   r   rL   rM   position_bias_num_bucketsnum_bucketsposition_bias_max_distancemax_distancesegment_typesnum_segmentsr   r   r   r   relative_attention_biasr!   s     r$   r   'CpmAntSegmentPositionEmbedding.__init__  s    33!;;"=="00')||KK$$v';';;f>^>^^**(
$r&   key_pos	query_poskey_segmentquery_segmentc           
      *   [         R                  " 5          UR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      UR                  S5      :w  a0  [        SUR                  S5       SUR                  S5       S35      eXcR                  S5      :w  d  XtR                  S5      :w  a!  [        SU SUR                  S5       S35      eXtR                  S5      :w  a!  [        SU SUR                  S5       S35      eUR	                  USU5      nUR	                  XWS5      nUR	                  USU5      nUR	                  XWS5      nU R                  XC5      nXR                  -   nU R                  [         R                  " U[         R                  UR                  S	9S S S 24   [         R                  " U[         R                  UR                  S	9S S 2S 4   -
  U R                  U R                  S
9n	[         R                  " X4:H  U	S S S 2S S 24   U5      nS S S 5        [        R                  " WU R                  5      n
U
R!                  SSSS5      R#                  5       n
U
$ ! , (       d  f       NS= f)Nr   r   z>key_pos.size(0) should be equal to query_pos.size(0), but got z and !z7keylen should be equal to key_segment.size(1), but got z;querylen should be equal to query_segment.size(1), but got r*   r0   re   )r   r   r   r+   )r   no_gradr.   r/   rf   !_segment_relative_position_bucketr   _position_bucketarangeint32re   r   whereF	embeddingr   rg   rq   )r"   r   r   r   r   batchkeylenquerylenrelative_position_bucketabsolute_position_bucketembedss              r$   r8   &CpmAntSegmentPositionEmbedding.forward  s|    ]]_LLOE\\!_F ~~a(H||A).."33$TU\UaUabcUdTeejktkykyz{k|j}}~  ))!,,<N<Nq<Q0Q$MfXUZ[f[k[klm[nZoopq  --a00$QRZQ[[`anasastuav`wwxy  ll5"f5G!u;I%**5"f=K)..uCM'+'M'Mm'i$'?BRBR'R$ (,'<'<V5;;?W?^?^_`dfg`gh,,xu{{C[CbCbcdegkdklm ,,!..	 (= ($ (-{{-(q!4(($C P 5t7S7ST1a+668W _s   H!J
Jc                 $    XR                   -  U-   $ r   )r   )r"   r   r   s      r$   r   @CpmAntSegmentPositionEmbedding._segment_relative_position_bucket  s    000;>>r&   c                 0   SnUS-  nUS:  R                  [        R                  5      U-  n[        R                  " U5      nUS-  nX:  nU[        R                  " UR                  5       U-  5      [        R                  " X5-  5      -  X%-
  -  R                  [        R                  5      -   n[        R                  " U[        R                  " XrS-
  5      5      nU[        R                  " XaR                  [        R                  5      U5      -  nU$ )Nr   r+   r   )
r1   r   r   abslogrp   rk   min	full_liker   )r"   relative_positionr   r   relative_buckets	max_exactis_smallrelative_position_if_larges           r$   r   /CpmAntSegmentPositionEmbedding._position_bucket  s    -155ekkB[P!II&781$	$0%.II'--/);<hh|/01&( "U[[/	&"
 &+YY&OO6aH&
" 	EKK2F2Fu{{2SUoppr&   )r   r   rM   r   r   )       )r:   r;   r<   r=   r   r   r   r?   r8   r   r   r@   rA   rB   s   @r$   r   r     sZ    
| 
22 <<2 \\	2
 ||2h?   r&   r   c                   z   ^  \ rS rSrU 4S jrS\R                  S\R                  S\R                  4S jrSrU =r	$ )CpmAntOutputi  c                 (  > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR
                  UR                  S9U l        [        R                  " UR                  5      U l        g )N)r   )r   r   r   rP   r   r   r   	LayerNormlayer_norm_epsrX   hidden_dropout_probrY   r!   s     r$   r   CpmAntOutput.__init__  s`    YYv779K9KL
f&8&8f>S>STzz&"<"<=r&   r'   input_tensorr   c                 p    U R                  U5      nU R                  U5      nU R                  X-   5      nU$ r   )r   rY   r  )r"   r'   r  s      r$   r8   CpmAntOutput.forward  s5    

=1]3}'CDr&   )r  r   rY   r   rB   s   @r$   r	  r	    s6    >U\\  RWR^R^  r&   r	  c                   `   ^  \ rS rSr% \\S'   Sr\R                  " 5       U 4S j5       r	Sr
U =r$ )CpmAntPreTrainedModeli  r   cpmantc                 $  > [         TU ]  U5        [        U[        5      (       a!  [        R
                  " UR                  5        g[        U[        5      (       a5  [        R                  " UR                  SU R                  R                  S9  gg)zInitialize the weightsg        )r4   stdN)r   _init_weightsr   r   initones_r    r   normal_r   r   init_std)r"   moduler#   s     r$   r  #CpmAntPreTrainedModel._init_weights  sc     	f%fo..JJv}}% >??LL77ct{{G[G[\ @r&   r   )r:   r;   r<   r=   r   __annotations__base_model_prefixr   r   r  r@   rA   rB   s   @r$   r  r    s)     
]]_] ]r&   r  c                      ^  \ rS rSrS\4U 4S jjrS rS rS r\	      SS\
R                  S-  S	\S-  S
\S-  S\S-  S\S-  S\S-  S\\
R                     \-  4S jj5       rSrU =r$ )CpmAntModeli  r   c                   > [         TU ]  U5        [        U5      U l        [        R
                  " UR                  UR                  5      U l        [        R
                  " UR                  UR                  UR                  -  -   UR                  5      U l        [        U5      U l        UR                  U l        UR                  U l	        U R                  5         g r   )r   r   r   encoderr   	Embeddingr   r   segment_embedding
vocab_sizeprompt_typesprompt_lengthinput_embeddingr   r_   	post_initr!   s     r$   r   CpmAntModel.__init__  s     $V,!#f.B.BFDVDV!W!|| 3 3f6J6J JJFL^L^ 
 <FC#11 ++r&   c                     U R                   $ r   r)  r"   s    r$   get_input_embeddings CpmAntModel.get_input_embeddings&  s    ###r&   c                     Xl         g r   r-  )r"   
embeddingsrr   s      r$   set_input_embeddings CpmAntModel.set_input_embeddings)  s    )r&   c                    UR                  S5      nUR                  S5      nUR                  n[        R                  " XgS9[        R                  " XgS9R	                  SS5      :*  nUS S 2S S S 24   US S 2S S 2S 4   R                  5       UR	                  SXf5      -  -  n	XS S 2S S S 24   US S 2S S 2S 4   :H  -  n	[        R                  " [        [        X`R                  -
  5      5      S S S2   US9S S S 24   R                  US5      US S 2S 4   :  n
[        R                  " [        R                  " XPR                  US9R                  5       U
4SS9n
U
R	                  XVS5      U
R	                  USU5      -  U	-  n	U	$ )Nr   r   )re   r*   rI   )r.   re   r   r   rf   logical_notrn   listr   r(  repeatcatonesr}   )r"   	input_idsspancontextlengthr   seqlenre   directional_mask_2dr^   mask_1ds              r$   _prepare_attention_mask#CpmAntModel._prepare_attention_mask,  sy   q!"!!#ll6AU\\RXEhEmEmnprsEtt D!,Aq$J++-0C0H0HF0[[
 (4
+;tAq$J?O+OP LLeF-?-?$?@A$B$GPVWX\^_X_`gghmopqQWo 	 ))UZZ/A/A&QVVXZabhij eQ7',,uaQW:XX[iir&   Nr;  r`   r   ra   rb   return_dictr   c           	      p   Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUR
                  [        R                  :w  a  UR                  [        R                  5      nUR
                  UR                  p[        R                  " US:g  SS5      R                  XS9n
U
S:g  R                  S5      R                  XS9n[        R                  " [        R                  " U R                  S-  U R                  -   U R                  S-  U R                  -   UU	S9R!                  UR#                  S5      S5      U4SS9nUR#                  5       u  p[        R                  " [        R$                  " XR                  XS9U
4SS9n
[        R&                  " X4SXS9n[        R                  " XU	S9R!                  US5      n[        R&                  " X4SXS9nU(       a  Uc  [)        U R                   S	9nUb  UR+                  5       OSnUR-                  5       nU R/                  U5      nU R1                  U
5      nUS:w  a  USS2SS2SS24   nUU-   nU R3                  UUX5      nU R5                  XX5      nUSS2US2SS24   nUSS2SS2US2SS24   nUSS2US2SS24   nU R7                  UUUUUUU5      u  nnnUS:X  a}  USS2U R                  S2SS24   nUb6  S
nU H,  nUUSS2SS2U R                  S2U R                  S24   4-  nM.     UnUb)  S
nU H  nUUSS2U R                  S2SS24   4-  nM!     UnU(       d  [9        S UUUU4 5       5      $ [;        UUUUS9$ )a9  
input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
Nr   r+   r   r*   r   r   rI   )r   r   c              3   .   #    U  H  oc  M  Uv   M     g 7fr   r   ).0vs     r$   	<genexpr>&CpmAntModel.forward.<locals>.<genexpr>  s      ^a^s   	)last_hidden_statera   r'   
attentions)r   r`   r   rD  rb   r0   r   r   r1   re   r   sumr9  r   r(  r&  r8  r.   zerosfullr	   get_seq_lengthrq   r)  r%  rB  r_   r#  tupler   )r"   r;  r`   r   ra   rb   rD  rr   r0   re   segmentr>  r   
seq_lengthr=  positionr<  past_lengthr'   segment_statesr^   r_   r   all_attentionsnew_attentions	attentionnew_hidden_stateshidden_states                               r$   r8   CpmAntModel.forward>  s   ( 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY!*!6IDKK<Q<Q	 ??ekk)!U[[1I!)9)9v++i1na366U6RQ,##B'***FII&&*T__<&&*T__<!	
 &*A. 
	 &NN,))U[[0B0B%_ahiopq**e0!5P<<
GNNuVWXzz5-qM0*$++>O:I:Uo446[\((*	,,Y7//8!+ArsAI6N%655iwW**8wP';<(:;%aKL!&;<%aq&89;?<< <
8(. !)!T-?-?-A1*DEM)!#!/I"yAt7I7I7KTM_M_Ma1a'b&ddN "0!/ ,$&!$5L%,q$:L:L:NPQ7Q*R)TT% %6$5! )?<M~^   '+++%	
 	
r&   )r#  r)  r_   r(  r%  r&  )NNNNNN)r:   r;   r<   r=   r   r   r/  r3  rB  r   r   r?   r}   r   rQ  r   r8   r@   rA   rB   s   @r$   r!  r!    s    | $*$  *.)-,0(,!%#'g
<<$&g
  $;g
 #Tk	g

 g
 $;g
 D[g
 
u||	6	6g
 g
r&   r!  zy
    The CPMAnt Model with a language modeling head on top (linear layer with weights tied to the input embeddings).
    )custom_introc                   "  ^  \ rS rSrSS0rS\4U 4S jjr\         SS\R                  S-  S\
S-  S	\S-  S
\S-  S\S-  S\R                  S-  S\S-  S\R                  S-  S\\R                  -  S\\-  4S jj5       rS rS rSrU =r$ )CpmAntForCausalLMi  zlm_head.weightzcpmant.input_embedding.weightr   c                    > [         TU ]  U5        [        U5      U l        [        R
                  " UR                  UR                  UR                  UR                  -  -   SS9U l
        U R                  5         g r   )r   r   r!  r  r   rP   r   r&  r'  r(  lm_headr*  r!   s     r$   r   CpmAntForCausalLM.__init__  sd     !&) yy 1 1F4G4G&J^J^4^ ^ej
 	r&   Nr;  ra   rb   r`   r   labelsrD  r^   logits_to_keepr   c
                 *   Ub  UOU R                   R                  nU R                  UUUUUU5      nU(       a  UR                  OUS   n[	        U	[
        5      (       a  [        U	* S5      OU	nU R                  USS2USS24   5      nSnUbA  [        5       nU" UR                  SUR                  S5      5      UR                  S5      5      nU(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  UR                  S9$ )u  
input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss.

Example:

Text Generation with CpmAntForCausalLM.
```python
>>> from transformers import CPMAntTokenizer, CpmAntForCausalLM

>>> texts = "今天天气不错，"
>>> model = CpmAntForCausalLM.from_pretrained("openbmb/cpm-ant-10b")
>>> tokenizer = CPMAntTokenizer.from_pretrained("openbmb/cpm-ant-10b")
>>> input_ids = tokenizer(texts, return_tensors="pt")
>>> outputs = model.generate(**input_ids)
>>> output_texts = tokenizer.batch_decode(outputs)
>>> print(output_texts)
['今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的']
```
Nr   r*   r   )losslogitsra   r'   rL  )r   rD  r  rK  r   intslicera  r   rf   r.   r   ra   r'   rL  )r"   r;  ra   rb   r`   r   rc  rD  r^   rd  rr   model_outputr'   slice_indicesrg  rf  	loss_funcoutputs                     r$   r8   CpmAntForCausalLM.forward  s   R &1%<k$++BYBY{{ 
 ;F66<XY?8B>SV8W8W~ot4]kmA}a,?@A(*IV[[V[[_=v{{2ODYab!11F)-)9TGf$EvE%(88&44#..
 	
r&   c                 .    U R                   R                  $ r   r  r)  r.  s    r$   r/  &CpmAntForCausalLM.get_input_embeddings  s    {{***r&   c                 $    XR                   l        g r   rp  )r"   r2  s     r$   r3  &CpmAntForCausalLM.set_input_embeddings  s    &0#r&   )r  ra  )	NNNNNNNNr   )r:   r;   r<   r=   _tied_weights_keysr   r   r   r   r?   r   r}   rh  rQ  r   r8   r/  r3  r@   rA   rB   s   @r$   r_  r_    s
    +,KL|   *.(,!%)-,0&*#'.2-.F
<<$&F
 F
 $;	F

  $;F
 #TkF
 t#F
 D[F
 t+F
 ell*F
 
'	'F
 F
P+1 1r&   r_  )r_  r!  r  )0r>   rk   r   torch.nn.functionalr   
functionalr   torch.nnr    r   r  activationsr   cache_utilsr   r	   
generationr
   modeling_outputsr   r   modeling_utilsr   utilsr   r   configuration_cpmantr   
get_loggerr:   loggerModuler   rD   r   r   r   r   r   r   r   r   r	  r  r!  r_  __all__r   r&   r$   <module>r     ss         % & ! . ) O - , . 
		H	%bii 2b#bii b#J3+ryy 3+l")) (		 4RYY 6*+RYY *+Z<@BII <@@ Y RYY Y z299  ]O ] ] N
' N
 N
b 
Z1- Z1
Z1z Hr&   