
    Z jP                       S r SSKrSSKrSSKJr  SSKrSSKJrJr  SSKJ	r	  SSK
Jr  SSKJrJrJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJrJrJr  SSKJr  \R<                  " \5      r S?S jr!S r"S?S jr#S r$\" SS9\ " S S\5      5       5       r%\" SS9\ " S S\5      5       5       r&\" SS9\ " S S\5      5       5       r'\" SS9\ " S S\5      5       5       r(\ " S  S!\5      5       r) " S" S#\RT                  5      r+ " S$ S%\RX                  5      r- " S& S'\RX                  5      r. " S( S)\RX                  5      r/ " S* S+\5      r0 " S, S-\5      r1\" S.S9 " S/ S0\)5      5       r2\" S1S9 " S2 S3\)5      5       r3\ " S4 S5\)5      5       r4\" S6S9 " S7 S8\)\5      5       r5\" S9S9 " S: S;\)\5      5       r6 " S< S=\)5      r7/ S>Qr8g)@zRPyTorch ProphetNet model, ported from ProphetNet repo(fairsequery_states version).    N)	dataclass)Tensornn)	LayerNorm   )ACT2FN)CacheDynamicCacheEncoderDecoderCache)GenerationMixin)GradientCheckpointingLayer)BaseModelOutput)PreTrainedModel)ModelOutputauto_docstringlogging   )ProphetNetConfigc                     U(       a,  [         R                  R                  U R                  5       US9$ [         R                  R                  X[        R
                  S9$ )Ndimr   dtype)r   
functionalsoftmaxfloattorchfloat32)hidden_stater   
onnx_traces      ڃ/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/prophetnet/modeling_prophetnet.pyr   r   %   sF    }}$$\%7%7%9s$CC}}$$\%--$PP    c                 h   [         R                  " XU 4X#S9[         R                  " U5      R                  -  nUR	                  5       R                  5       n[        U5       H,  nXV   R                  SSS9  XF   R                  U* S-   5        M.     SUSS2SS2S4'   [         R                  " XE/SS9$ )	z8
This function computes the bias for the predict stream
)devicer   r   F)wrapr   N   r   )
r   onesfinfomindetachclonerangefill_diagonal_triu_cat)sequence_lengthngramr$   r   
left_blockright_block
stream_idxs          r!   ngram_attention_biasr5   ,   s    
 	

EO<VY\a\g\ghm\n\r\rr  ##%++-KEl
..qu.=$$j[1_5 # Jq!Qw99j.A66r"   c                    U* nSnU(       a[  U S-  n U[         R                  " U[         R                  " U5      5      R                  5       U -  -   n[         R                  " U5      nO+[         R
                  " U[         R                  " U5      5      nU S-  n[         R                  " XF5      nU[         R                  " UR                  5       U-  5      [        R                  " X-  5      -  X-
  -  -   n[         R                  " U[         R                  " U5      U S-
  -  5      R                  5       nU[         R                  " XtR                  5       U5      -   nU$ )zg
This function computes individual parts of the relative position buckets. For more detail, see paper.
r   r&   r   )r   lt
zeros_likeintabsmaxlogr   mathr)   	ones_likewhere)	num_bucketsmax_distancerelative_positionsis_bidirectionalinv_relative_positionsrel_positions_bucket	max_exactis_smallval_if_larges	            r!   compute_relative_bucketsrI   =   s>    10!Q& hh-u/?/?@V/WX\\^allm 	 "'+A!B!&+A5CSCSTjCk!lq Ixx.:Huyy)?)E)E)G))STW[W_W_ X  		  " "L 99\5??<+HKZ[O+\]aacL/%++hHbHbHdfr2ssr"   c                 x   UR                  S5      R                  SUR                  S5      S5      nX2R                  S5      -
  n[        R                  " US-
  U4SS9R                  S5      nUR                  SUR                  S5      S5      nXBR                  S5      -
  n[        XUSS9n[        XUSS9nXV4$ )ze
This function computes both main and predict relative position buckets. For more detail, see paper.
r   r   F)rC   )	unsqueezerepeatsizer   r/   rI   )r@   rA   position_idsmain_stream_relative_positions$predicting_stream_relative_positionsmain_relative_position_buckets!predict_relative_position_bucketss          r!   #compute_all_stream_relative_bucketsrT   X   s    
 &2%;%;A%>%E%EaIZIZ[]I^`a%b"%CF\F\]_F`%`" ,199lQ6F5U[]+^+h+hij+k(+O+V+VWXZfZkZklnZoqr+s(+ORhRhikRl+l( &>#ATY&" )A#GZ_)% *LLr"   zF
    Base class for sequence-to-sequence language models outputs.
    )custom_introc                      \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\R                  S-  \S'   Sr\S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S	'   Sr\\R                     S-  \S
'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S'   Sr\R                  S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S'   Srg)ProphetNetSeq2SeqLMOutputo   ar  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss.
logits (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, config.vocab_size)`):
    Prediction scores of the main stream language modeling head (scores for each vocabulary token before
    SoftMax).
logits_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
    Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
    SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
    shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

    Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
    outputs.
decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
    decoder_sequence_length, decoder_sequence_length)`.

    Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
    weighted average in the self-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
    Sequence of hidden-states at the output of the last layer of the encoder of the model.
Nlosslogitslogits_ngrampast_key_valuesdecoder_hidden_statesdecoder_ngram_hidden_statesdecoder_attentionsdecoder_ngram_attentionscross_attentionsencoder_last_hidden_stateencoder_hidden_statesencoder_attentions )__name__
__module____qualname____firstlineno____doc__rY   r   FloatTensor__annotations__rZ   r[   r\   r	   r]   tupler^   r_   r`   ra   rb   rc   rd   __static_attributes__re   r"   r!   rW   rW   o   s)   < &*D%

d
")'+FE$+-1L%##d*1$(OUT\(=A5!2!23d:ACGu'8'8!9D!@G:>e//047>@DeE$5$56=D8<eE--.5<:>u0047>=A5!2!23d:A:>e//047>r"   rW   z
    Base class for model encoder's outputs that also contains : pre-computed hidden states that can speed up sequential
    decoding.
    c                      \ rS rSr% Sr\R                  \S'   Sr\R                  S-  \S'   Sr	\
S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S	'   Sr\\R                     S-  \S
'   Sr\\R                     S-  \S'   Sr\R                  S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S'   Srg)ProphetNetSeq2SeqModelOutput   ad  
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
    Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.

    If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
    hidden_size)` is output.
last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size,ngram * decoder_sequence_length, config.vocab_size)`, *optional*):
    Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
    shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

    Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
    outputs.
decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
    decoder_sequence_length, decoder_sequence_length)`.

    Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
    weighted average in the
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
    Sequence of hidden-states at the output of the last layer of the encoder of the model.
last_hidden_stateNlast_hidden_state_ngramr\   r]   r^   r_   r`   ra   rb   rc   rd   re   )rf   rg   rh   ri   rj   r   rk   rl   rs   r\   r	   r]   rm   r^   r_   r`   ra   rb   rc   rd   rn   re   r"   r!   rp   rp      s   : (((8<U..5<$(OUT\(=A5!2!23d:ACGu'8'8!9D!@G:>e//047>@DeE$5$56=D8<eE--.5<:>u0047>=A5!2!23d:A:>e//047>r"   rp   zs
    Base class for model's outputs that may also contain a past key/values (to speed up sequential decoding).
    c                   Z   \ rS rSr% Sr\R                  \S'   Sr\R                  S-  \S'   Sr	\
S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S	'   Sr\\R                     S-  \S
'   Sr\\R                     S-  \S'   Srg)ProphetNetDecoderModelOutput   as  
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
    Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.

    If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
    hidden_size)` is output.
last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
    Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
hidden_states_ngram (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
    shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

    Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
    outputs.
ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
    decoder_sequence_length, decoder_sequence_length)`.

    Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
    weighted average in the
rr   Nrs   r\   hidden_stateshidden_states_ngram
attentionsngram_attentionsra   re   )rf   rg   rh   ri   rj   r   rk   rl   rs   r\   r	   rw   rm   rx   ry   rz   ra   rn   re   r"   r!   ru   ru      s    6 (((8<U..5<$(OUT\(59M5**+d29;?u001D8?26Je''(4/68<eE--.5<8<eE--.5<r"   ru   c                      \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\R                  S-  \S'   Sr\S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S	'   Sr\\R                     S-  \S
'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S'   Srg)ProphetNetDecoderLMOutput   aF	  
ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
    shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

    Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
    outputs.
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss.
logits (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, config.vocab_size)`):
    Prediction scores of the main stream language modeling head (scores for each vocabulary token before
    SoftMax).
logits_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
    Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
    SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
hidden_states_ngram (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
    shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

    Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
    outputs.
ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
    decoder_sequence_length, decoder_sequence_length)`.

    Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
    weighted average in the
NrY   rZ   r[   r\   rw   rx   ry   rz   ra   re   )rf   rg   rh   ri   rj   rY   r   rk   rl   rZ   r[   r\   r	   rw   rm   rx   ry   rz   ra   rn   re   r"   r!   r|   r|      s     D &*D%

d
")'+FE$+-1L%##d*1$(OUT\(59M5**+d29;?u001D8?26Je''(4/68<eE--.5<8<eE--.5<r"   r|   c                   .    \ rS rSr% \\S'   SrSrS rSr	g)ProphetNetPreTrainedModeli4  config
prophetnetTc                    U R                   R                  nU R                   R                  nUc   S5       eUR                  UR                  5      nUSS S24   R                  5       USSS 24'   X$S'   Uc   S5       eUR                  US:H  U5        [        R                  " US:  5      R                  5       (       d   S	5       eU$ )
Nzself.model.config.decoder_start_token_id has to be defined. In ProphetNet it is usually set to the pad_token_id. See ProphetNet docs for more information.rK   r   ).r   z1self.model.config.pad_token_id has to be defined.r   z8Verify that `shifted_input_ids` has only positive values)
r   decoder_start_token_idpad_token_id	new_zerosshaper+   masked_fill_r   allitem)self	input_idsr   r   shifted_input_idss        r!   _shift_right&ProphetNetPreTrainedModel._shift_right:  s    !%!C!C{{//%1 	
F	
1 &//	@%.sCRCx%8%>%>%@#qr'"$:&!'\)\\'&&'8D'@,Oyy*a/05577s9ss7  r"   re   N)
rf   rg   rh   ri   r   rl   base_model_prefixsupports_gradient_checkpointingr   rn   re   r"   r!   r   r   4  s    $&*#!r"   r   c                   T   ^  \ rS rSrSrS\SS4U 4S jjrS
U 4S jjrU 4S jrS	r	U =r
$ )ProphetNetPositionalEmbeddingsiQ  z
This module learns positional embeddings up to a fixed maximum size. Padding ids are ignored by either offsetting
based on padding_idx or by setting padding_idx to None and ensuring that the appropriate position ids are passed to
the forward function.
r   returnNc                    > UR                   U l        [        TU ]  UR                   UR                  UR
                  5        g N)max_position_embeddings
max_lengthsuper__init__hidden_sizer   r   r   	__class__s     r!   r   'ProphetNetPositionalEmbeddings.__init__X  s3     88779K9KVM`M`ar"   c                 D  > Ub  U R                   b   S5       eUc  Ubj  UR                  5       S:w  aV  UR                  5       nUS   U-   n[        R                  " S[        R                  US9[        U R                   U-   5      -  nOUc$  [        R                  " U[        R                  US9n[        R                  " USS9R                  U5      U-  R	                  5       U R                   -   nUR                  SU R                  S-
  5      n[        TU ]-  U5      U4$ )NzCIf position_ids is pre-computed then padding_idx should not be set.r   r   )r   r   r   r$   r   )padding_idxget_seq_lengthr   r'   longr9   cumsumtype_asclampr   r   forward)	r   inputs_shaper$   attention_maskr\   rO   prev_num_input_idsnum_input_idsr   s	           r!   r   &ProphetNetPositionalEmbeddings.forward\  s   $$*:*:*B 	
Q	
C */M/M/OST/T &5%C%C%E" ,Q2D D$zz&

6R((=89  ")%*ZZEJJW]%^N LLQ7??OR``$&4++ ,
  ,11!T__q5HIw|,l::r"   c                 "   > [         TU ]  U5      $ r   )r   r   )r   rO   r   s     r!   _forward'ProphetNetPositionalEmbeddings._forwardx  s    w|,,r"   )r   )NNN)rf   rg   rh   ri   rj   r   r   r   r   rn   __classcell__r   s   @r!   r   r   Q  s.    b/ bD b;8- -r"   r   c                      ^  \ rS rSrSrSS\S\S\S-  4U 4S jjjr    SS\S-  S	\S-  S
\	S-  S\
S-  S\\\S-  4   4
S jjrSrU =r$ )ProphetNetAttentioni|  z=Multi-headed attention from 'Attention Is All You Need' paperNr   num_attn_heads	layer_idxc                   > [         TU ]  5         UR                  nUR                  U l        UR                  U l        X l        XB-  U l        X0l        U R                  U-  U:X  d   S5       e[        R                  " XD5      U l
        [        R                  " XD5      U l        [        R                  " XD5      U l        [        R                  " XD5      U l        g )Nzw`config.hidden_size` must be divisible by `config.num_encoder_attention_heads` and `config.num_decoder_attention_heads`)r   r   r   attention_dropoutdropoutr   head_dimr   r   Linearkey_proj
value_proj
query_projout_proj)r   r   r   r   r   r   s        r!   r   ProphetNetAttention.__init__  s    ((!'!9!9~~,#5"}}~-< 	
4	
<
 		+;))K=))K=		+;r"   key_value_statesr   r\   output_attentionsr   c                    UR                  5       u  pxn	US Ln
[        UR                  5       5      UUU	/:X  d   SXxU	4 SUR                  5        35       eU R                  U5      U R                  S-  -  nSnUb]  [	        U[
        5      (       aF  UR                  R                  U R                  5      nU
(       a  UR                  nOUR                  nOUnU
(       a  UOUnU
(       aQ  UbN  U(       aG  WR                  U R                     R                  nUR                  U R                     R                  nOU R                  U5      nU R                  U5      nUR!                  USU R"                  U R                  5      R%                  SS5      nUR!                  USU R"                  U R                  5      R%                  SS5      nUbU  WR'                  UUU R                  5      u  nnU
(       a.  [	        U[
        5      (       a  SUR                  U R                  '   UR!                  XxU R"                  U R                  5      R%                  SS5      nUR                  S5      n[(        R*                  " S	XR%                  SS
5      5      nXpR"                  UU4nUR                  5       U:w  a  [-        SU SUR                  5        35      eUb  UR/                  5       S:X  a  S nXpR"                  SU4nUb3  UR                  5       U:w  a  [-        SU SUR                  5        35      eUb  UU-   nU(       a  UnOS n[0        R2                  R5                  USS9n[0        R2                  R7                  UU R8                  U R:                  S9n[(        R*                  " S	UU5      nXpR"                  XR                  4nUR                  5       U:w  a  [-        SU SUR                  5        35      eUR%                  SS5      R=                  XxU	5      nU R?                  U5      n[0        R2                  R7                  UU R6                  U R:                  S9nUU4$ )Nz Size of hidden states should be z	, but is       ?FrK   r   r&   Tzbsij,bsjk->bsikr   z#Attention weights should have size r   z Attention mask should have size r   ptrainingz `attn_output` should have shape , but is of shape ) rN   listr   r   
isinstancer   
is_updatedgetr   cross_attention_cacheself_attention_cachelayerskeysvaluesr   r   viewr   	transposeupdater   einsum
ValueErrorr   r   r   r   r   r   r   reshaper   )r   rw   r   r   r\   r   kwargs
batch_sizetgt_lenr   is_cross_attentionquery_statesr   curr_past_key_valuescurrent_states
key_statesvalue_statessrc_lenattn_weightsexpected_shapeattn_weights_reshaped
attn_probsattn_outputs                          r!   r   ProphetNetAttention.forward  s0    ,9+=+=+?(
[ .T9M&&().
 
 	p .j;.N-OyYfYkYkYmXno		p 
 }59KL
&/+>??,77;;DNNK
%+:+P+P(+:+O+O('6$-?)]/"=*-44T^^DIIJ/66t~~FMML~6J??>:L#R9L9Ldmm\ffghjklJ',,ZT=P=PRVR_R_`jjklnopL*+?+F+FzS_aeaoao+p(
L%*_FY*Z*ZAEO..t~~>#((d>Q>QSWS`S`akklmopq//!$||$5|EYEYZ[]^E_`$&9&97GL.0B>BRR[\h\m\m\o[pqrr %.*<*<*>!*C!N$&9&91gF%.*=*=*?>*Q??OyYgYlYlYnXopqq%'.8L$0!$(!}},,\r,B]]**$$]] + 


 ll#4j,O$&9&97MMR/??OOabmbrbrbtauvww!++Aq199*{[mmK0mm++K4<<RVR_R_+`111r"   )	r   r   r   r   r   r   r   r   r   r   )NNNF)rf   rg   rh   ri   rj   r   r9   r   r   r	   boolrm   r   rn   r   r   s   @r!   r   r   |  s    G</ < <QTW[Q[ < <0 +/(,(,).[2 !4-[2 	[2
 [2  $;[2 
vv}$	%[2 [2r"   r   c                   >   ^  \ rS rSrSrS\S\4U 4S jjrS rSr	U =r
$ )ProphetNetFeedForwardi  ze
This is the residual two feed-forward layer block based on the original Transformer implementation.
r   ffn_dimc                 ,  > [         TU ]  5         [        UR                     U l        [
        R                  " UR                  U5      U l        [
        R                  " X!R                  5      U l	        UR                  U l
        UR                  U l        g r   )r   r   r   activation_functionactivation_fnr   r   r   intermediateoutputactivation_dropoutr   )r   r   r   r   s      r!   r   ProphetNetFeedForward.__init__  si    #F$>$>?IIf&8&8'Bii););<"(";";~~r"   c                 4   U R                  U5      nU R                  U5      n[        R                  R	                  XR
                  U R                  S9nU R                  U5      n[        R                  R	                  XR                  U R                  S9nU$ )Nr   )r   r   r   r   r   r   r   r   )r   rw   s     r!   r   ProphetNetFeedForward.forward  s    ))-8**=9--m?V?Vaeanan-oM2--m||VZVcVc-dr"   )r   r   r   r   r   )rf   rg   rh   ri   rj   r   r9   r   r   rn   r   r   s   @r!   r   r     s&    &/ &# & r"   r   c                   p   ^  \ rS rSrSS\4U 4S jjjrS rS r      SS\S-  4S jjr	S	 r
S
 rSrU =r$ )ProphetNetNgramSelfAttentioni	  Nr   c                   > [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        UR
                  U l        UR                  U l        UR                  U l        UR                  U R                  -  U l	        UR                  U l
        X l        U R                  U R                  -  UR                  :X  d   S5       e[        R                  " UR                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  U R                  U R                  -  5      U l        SU l        g )Nz6config.hidden_size must be divisible by num_attn_headsF)r   r   r   r@   relative_max_distancenum_decoder_attention_headsr   r   r   r   r1   r   r   r   r   r   r   r   relative_pos_embeddingsr    r   r   r   r   s      r!   r   %ProphetNetNgramSelfAttention.__init__
  sf   !--!--%+%A%A"$@@~~!'!9!9**d.A.AA\\
"}}t222f6H6HH 	
D	
H 		&"4"4f6H6HI))F$6$68J8JK))F$6$68J8JK 		&"4"4f6H6HI (*yy1C1CTEUEUX\XkXkEk'l$  r"   c                     UR                  X2U R                  U R                  5      R                  SS5      R	                  5       $ Nr   r&   )r   r   r   r   
contiguous)r   tensorseq_lenr   s       r!   _shape#ProphetNetNgramSelfAttention._shape(  s7    {{:0C0CT]]S]]^_abcnnppr"   c                     SU l         g )NT)r    r   s    r!   prepare_for_onnx_export_5ProphetNetNgramSelfAttention.prepare_for_onnx_export_+  s	    r"   r\   c           	         UR                  5       u  pn[        UR                  5       5      XU/:X  d   SXU4 SUR                   35       eU R                  U5      nU R	                  U5      nU R                  U5      nXR                  S-  -  nU R                  XU	5      nU R                  USU	5      nU R                  USU	5      nXR                  SU R                  4nUR                  " U6 nUR                  " U6 nUR                  " U6 nUR                  SU R                  -   SS9nUR                  SU R                  -   SS9nUR                  SU R                  -   SS9nUR                  SU R                  -   SS9nUS   USS  nnUS   USS  nnUS   USS  nnUS   USS  nnUbD  [        U[        5      (       a  UR                  nOUnUR                  UUU R                   5      u  nnU
SU R                  -   -  n["        R$                  " S	UUR'                  SS
5      5      nU R)                  UUXu5      nUU-   nUb  UU-   n[+        USU R,                  S9R/                  U5      n [0        R2                  R5                  U U R6                  U R8                  S9n ["        R$                  " S	U U5      n!U!R'                  SS5      R                  U	SUU5      n!U R;                  U!5      n!["        R<                  " US5      R?                  XR                  U R                  UU R                  5      n"["        R<                  " U V#s/ s H  n#["        R@                  " UU#/S5      PM     sn#S5      n$["        R<                  " USS9n%["        R@                  " U V&s/ s H+  n&["        R@                  " UU&/S5      RC                  S5      PM-     sn&S5      n'["        R$                  " SU"U$45      n(U RE                  U%U(Xv5      n)U(U)-   n(Ub5  URG                  SSSS
S5      nURI                  U(RJ                  5      nU(U-   n([+        U(SU R,                  S9R/                  U(5      n*[0        R2                  R5                  U*U R6                  U R8                  S9n*["        R$                  " SU*U'R'                  SS5      45      n+U+R'                  SS
5      n+U+R                  XR                  UU5      n+U R;                  U+5      n+["        R@                  " U!U+/S5      R?                  U	SU5      n,U R?                  XR                  US5      n [0        R2                  R5                  U,U R4                  U R8                  S9n,U,U U*4$ s  sn#f s  sn&f )Nz#`hidden_states` should be of shape r   r   rK   r   r   r&   r   zbntc,bncs->bntsr   )r   r    r   zbnhtc,bnhsc->bnhts   zbnhts,bnhsc->bnhtc)&rN   r   r   r   r   r   r   r  r   r   chunkr1   r   r   r   r   r   r   r   r    get_main_relative_pos_embeddingsr   r    r   r   r   r   r   r   r   stackr   r/   rL   #get_predict_relative_pos_embeddingspermutetor   )-r   rw   r\   r   extended_predict_attention_maskrR   rS   rO   r   r   ngram_sequence_lengthr   r   r   r   
proj_shapehidden_states_listquery_states_listkey_states_listvalue_states_listmain_hidden_stateshidden_states_predict_listmain_query_statespredict_query_states_listmain_key_statespredict_key_states_listmain_value_statespredict_value_states_listr   r0   main_attn_weightsmain_relative_pos_embeddingsmain_attn_probsmain_attn_outputpredict_query_stateskeypredict_key_statespredict_hidden_statesv_ppredict_value_statespredict_attn_weightspredict_relative_pos_embeddingspredict_attn_probspredict_attn_outputr   s-                                                r!   r   $ProphetNetNgramSelfAttention.forward.  s    :G9K9K9M6
;M&&()jQ\-]] 	
1*U`2`1a b##$&	
] }5]]=1
}5 $}}c'9: {{<
S[[R<
{{<Z@ "5"5r4==I
#++Z8''4
#++Z8 +00TZZQ0G(..q4::~1.E$**1tzz>q*A(..q4::~1.E9KA9NPbcdcePf67H7KM^_`_aMb43B13EWXWYGZ07H7KM^_`_aMb4 &/+>??'6'K'K$'6$1E1L1L!2DNN2.O.
 0A

NC "LL):<MOhOhijlmOno (,'L'L 1<(
$ .0LL% 1N B!
 '#
$	 	 --//4CYCYdhdqdq/r
 !<<(9?L]^+55a;CCJPQSbdop==)9:  %{{+DaHMM

D$7$7$-- 

 #[[Zq)rZqSV%))_c4JA*NZq)rtuv !&,FA N  %yyLefLeSUYY)3/3==a@Lefhi 
  %||,@CWYkBlm +/*R*R!#7+
'
  46UU*6.M.U.UVWYZ\]_`bc.d+.M.P.PQeQkQk.l+#7:Y#Y $ 
 '&
'	 	  ]]22$"8"84== 3 
 $ll #57K7U7UVWYZ7["\
 2;;AqA199*jjRacno"mm,?@ ii!13F GKPPQ[]_alm)..z;N;NP_acdmm++K4<<RVR_R_+`O-???{ *s gs   
#V 2Vc                    UR                   u  pVpxUR                  XVXx5      nUc  UR                   S S u  pY[        R                  " SUR                   S   S-   5      R	                  S5      R	                  S5      R                  XYS5      R                  UR                  5      n
XR	                  S5      R                  XYS5      -
  n
[        U R                  U R                  U
S5      nU R                  U5      nUR                  UR                   S S U R                  U R                  4-   5      nUR                  SSSS5      nUR                  UR                   S S S-   5      nUR                  SU R                  S5      nUR                  SUR                   S   5      nUR                  5       nUR                  SUR!                  S5      5      n[        R"                  " USUS9nUR                  XVUS5      nU$ )	Nr&   r   rK   r   Fr   )rK   r   index)r   r   r   arangerL   rM   r  r$   rI   r@   r   r   r   r  r   r   rN   gather)r   rw   r   rO   rR   r   r   r   r   r0   rB   rel_pos_embeddingsr  s                r!   r
  =ProphetNetNgramSelfAttention.get_main_relative_pos_embeddings  s    8D7I7I4
G#((WV)1*7*=*=bq*A'JQ 2 22 6 :;11
Q7L''(  "46L6LQ6O6V6VWatu6v!v-E  $"<"<>PRW.*
 "99-H/44$$Ra(D,<,<d>Q>Q+RR
 0771aC/778J8J2A8NQV8VW)G)N)NqRVReRegh)i&)G)L)L.44R8*
& *H)L)L)N&/77<N<S<STV<WX',||4FAUs't$'C'H'Helnp'q$++r"   c                     UR                   SS u  pVUc  UR                   S   nUS   S   US-
  :X  d   S5       e[        R                  " SU5      R                  S5      R                  S5      R	                  XVS5      R                  UR                  5      nXR                  S5      R	                  XVS5      -
  n[        U R                  U R                  US5      nUR                  SS5      nU R                  U5      n	U	R                  UR                   S S U R                  U R                  4-   5      n	U	R                  SSSSS5      n	U	R                  SU R                  5      n	UR                  S5      nUR	                  U R                   SU R                  S5      nUR                  SUR#                  S5      5      R%                  5       n[        R&                  " U	SUS	9n
U
R                  XPR                   U R                  US5      n
U
$ )
Nr   r&   rK   r   zb`position_ids` are incorrect. They should be of the format 1 2 3 4 5 ... (key_sequence_length - 1)Fr  r   r.  )r   r   r0  rL   rM   r  r$   rI   r@   r   r   r   r   r   r  r   r1   rN   r   r1  )r   rw   r   rO   rS   r   r0   key_sequence_lengthrB   r2  r)  s              r!   r  @ProphetNetNgramSelfAttention.get_predict_relative_pos_embeddings  s     '4&9&9!A&>#
,4"."4"4R"8?1%)<q)@@ t@ Q 3411
Q7L''(  "46L6LQ6O6V6VWatu6v!v0H  $"<"<>PRW1-
 &//15!99-H 044$(8(8$:M:M'NN
 0771aAF/77D<L<LM,M,W,WXY,Z),M,T,TJJ4..-
) -N,R,R166r:-

$& 	* +0,,A-N+
'
 +J*N*N

D$7$7"+
' /.r"   )r   r   r   r   r   r   r1   r   r@   r    r   r   r   r   r   r   NNNNNN)rf   rg   rh   ri   r   r   r  r  r	   r   r
  r  rn   r   r   s   @r!   r   r   	  s\     /    <q )-(,'+*.a@ a@F+,Z9/ 9/r"   r   c                   H   ^  \ rS rSrSrS\4U 4S jjr SS\4S jjrSr	U =r
$ )	ProphetNetEncoderLayeri:  z
Encoder block for Prophetnet
r   c                    > [         TU ]  5         [        XR                  5      U l        [        UR                  5      U l        [        XR                  5      U l
        [        UR                  5      U l        g r   )r   r   r   num_encoder_attention_heads	self_attnr   r   self_attn_layer_normr   encoder_ffn_dimfeed_forwardfeed_forward_layer_normr   s     r!   r   ProphetNetEncoderLayer.__init__?  s[    ,V5W5WX$-f.@.@$A! 2&:P:PQ'01C1C'D$r"   r   c                     U R                  UUUS9u  pEU R                  XA-   5      nU R                  U5      nU R                  Xa-   5      nU4nU(       a  Xu4-  nU$ )N)rw   r   r   )r<  r=  r?  r@  )r   rw   r   r   attention_outputr   feed_forward_outputoutputss           r!   r   ProphetNetEncoderLayer.forwardI  s{     *.')/ *8 *
&
 112B2RS #//>445H5XY "&Gr"   )r?  r@  r<  r=  Frf   rg   rh   ri   rj   r   r   r   r   rn   r   r   s   @r!   r9  r9  :  s0    E/ E #(	  	 r"   r9  c                   n   ^  \ rS rSrSrS
S\4U 4S jjjr          SS\S-  S\S-  4S jjrS	r	U =r
$ )ProphetNetDecoderLayeric  z
Decoder block for Prophetnet
Nr   c                 b  > [         TU ]  5         [        XS9U l        [	        UR
                  5      U l        UR                  (       a3  [        XR                  US9U l
        [	        UR
                  5      U l        [        XR                  5      U l        [	        UR
                  5      U l        g )Nr   )r   r   r   r<  r   r   r=  add_cross_attentionr   r   
cross_attncross_attn_layer_normr   decoder_ffn_dimr?  r@  r   s      r!   r   ProphetNetDecoderLayer.__init__h  s    5fR$-f.@.@$A! %%1&:\:\hqrDO)263E3E)FD& 2&:P:PQ'01C1C'D$r"   	use_cacher   c           
      &   U R                  UU	UUUUUS9u  pnU R                  X-   5      nS nUb*  U R                  UUUU	US9u  nnU R                  UU-   5      nU R	                  U5      nU R                  UU-   5      nU4nU(       a  UXU4-  nU$ )N)rw   r\   r   r  rR   rS   rO   )rw   r   r   r\   r   )r<  r=  rN  rO  r?  r@  )r   rw   r   rc   encoder_attn_maskr  rR   rS   rO   r\   rR  r   r   ngram_attention_outputself_attn_weightsself_attn_weights_ngramcross_attn_weightsrC  rD  rE  s                       r!   r   ProphetNetDecoderLayer.forwardw  s      NR^^'+),K+I.O% N\ N
J3J 11-2XY! ,37??+!60 /"3 4C 400 !667G-7WXM #//>445H=5XY ")DVWWGr"   )rN  rO  r?  r@  r<  r=  r   )
NNNNNNNNTFrH  r   s   @r!   rJ  rJ  c  sc    E/ E E$ "(,'+*.!%).0 $;0  $;0 0r"   rJ  z=
    The standalone encoder part of the ProphetNetModel.
    c                      ^  \ rS rSrS\4U 4S jjrS rS r\      SS\	R                  S-  S\	R                  S-  S	\	R                  S-  S
\S-  S\S-  S\S-  S\\-  4S jj5       rSrU =r$ )ProphetNetEncoderi  r   c                   > [         TU ]  U5        [        R                  " UR                  UR
                  UR                  S9U l        [        U5      U l	        [        UR
                  5      U l        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l        U R%                  5         g s  snf )Nr   F)r   r   r   	Embedding
vocab_sizer   r   word_embeddingsr   position_embeddingsr   embeddings_layer_norm
ModuleListr,   num_encoder_layersr9  r   gradient_checkpointing	post_init)r   r   _r   s      r!   r   ProphetNetEncoder.__init__  s     !||F,=,=v?Q?Q_e_r_rs#A&#I %.v/A/A%B"mmUSYSlSlMm$nMm%;F%CMm$no&+#	 %os   Cc                     U R                   $ r   r`  r  s    r!   get_input_embeddings&ProphetNetEncoder.get_input_embeddings      ###r"   c                     Xl         g r   rj  r   values     r!   set_input_embeddings&ProphetNetEncoder.set_input_embeddings      $r"   Nr   r   inputs_embedsr   output_hidden_statesreturn_dictr   c                 2   Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUc  Uc  [	        S5      eUb  Ub  [	        S5      eUb  Uc  U R                  U5      nUb}  SUSS2SSSS24   R                  SU R                   R                  SS5      -
  [        R                  " U R                  5      R                  -  nUR                  UR                  5      nOSnU R                  UR                  SS UR                  5      u  pX9-   nU R!                  U5      n["        R$                  R'                  XR                   R&                  U R(                  S9nU(       a  SOSnU(       a  SOSn[+        U R,                  5       H0  u  pU(       a  X4-   nU" UUUS	9nUS
   nU(       d  M'  UUS   4-   nM2     U(       a  X4-   nU(       d  [/        S XU4 5       5      $ [1        XUS9$ )a  
Example:

```python
>>> from transformers import AutoTokenizer, ProphetNetEncoder
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
>>> model = ProphetNetEncoder.from_pretrained("patrickvonplaten/prophetnet-large-uncased-standalone")
>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> last_hidden_states = outputs.last_hidden_state
```Nz3Either input_ids or inputs_embeds has to be passed.z2Make sure to only pass input_ids or inputs_embeds.      ?r   r&   r   re   )r   r   r   c              3   .   #    U  H  oc  M  Uv   M     g 7fr   re   .0vs     r!   	<genexpr>,ProphetNetEncoder.forward.<locals>.<genexpr>  s     l$Zq$Z   	)rr   rw   ry   )r   r   ru  rv  r   r`  rM   r;  r   r(   r   r)   r  ra  r   r$   rb  r   r   r   r   	enumerater   rm   r   )r   r   r   rt  r   ru  rv  r   extended_attention_maskra  rO   rw   rc   all_attentionsidxencoder_layerlayer_outputss                    r!   r   ProphetNetEncoder.forward  s-   4 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY!6RSS"}'@QRR"}'< 00;M %nQdA%56==aAhAhjkmnooDJJ'++',# '>&@&@ATAT&U#&*#,0,D,D]EXEXY[Z[E\^k^r^r,s)%;22=A--m{{?R?R]a]j]j-k&:0d"+DKK"8C#(=@P(P%)6"3M *!,M  !/=3C2E!E #9  $9<L$L!l]>$Zlll+]k
 	
r"   )rb  re  r   ra  r`  r7  )rf   rg   rh   ri   r   r   rk  rq  r   r   r   r   rm   r   r   rn   r   r   s   @r!   r[  r[    s    / $%  *..2-1)-,0#'N
<<$&N
 t+N
 ||d*	N

  $;N
 #TkN
 D[N
 
	 N
 N
r"   r[  z=
    The standalone decoder part of the ProphetNetModel.
    c                   L  ^  \ rS rSrS\4U 4S jjrS rS r\          SS\	R                  S-  S\	R                  S-  S	\	R                  S-  S
\	R                  S-  S\S-  S\	R                  S-  S\S-  S\S-  S\S-  S\S-  S\\-  4S jj5       rS rS rS rSrU =r$ )ProphetNetDecoderi  r   c           
        > [         TU ]  U5        UR                  U l        UR                  U l        UR                  U l        UR
                  U l        UR                  U l        [        R                  " UR                  UR                  UR                  S9U l        [        U5      U l        [        R                  " U R                  UR                  S 5      U l        [        R"                  " [%        UR&                  5       Vs/ s H  n[)        XS9PM     sn5      U l        [-        UR                  5      U l        SU l        U R3                  5         g s  snf )Nr]  rL  F)r   r   r1   r@   r   r   r   max_target_positionsr   r^  r_  r   r   r`  r   ra  ngram_embeddingsrc  r,   num_decoder_layersrJ  r   r   rb  re  rf  )r   r   ir   s      r!   r   ProphetNetDecoder.__init__  s    \\
!--%+%A%A"~~$*$B$B!!||F,=,=v?Q?Q_e_r_rs#A&#I  "TZZ9K9KT RmmBGHaHaBbcBbQ#F8Bbc
 &/v/A/A%B"&+# ds   Ec                     U R                   $ r   rj  r  s    r!   rk  &ProphetNetDecoder.get_input_embeddings1  rm  r"   c                     Xl         g r   rj  ro  s     r!   rq  &ProphetNetDecoder.set_input_embeddings4  rs  r"   Nr   r   rc   encoder_attention_maskr\   rt  rR  r   ru  rv  r   c                 
   Ub  UOU R                   R                  nUb  UOU R                   R                  nU	b  U	OU R                   R                  n	U
b  U
OU R                   R                  n
Uc  Uc  [        S5      eUb  Ub  [        S5      eUb  Uc  U R                  U5      nUR                  SS u  pU R                  (       a/  U R                  (       a  U(       a  [        R                  S5        SnU(       ab  Uc_  Uc  U R                   R                  (       a.  [        [        U R                   S9[        U R                   S95      O[        U R                   S9nUb  UR                  5       OSnU R!                  X4UR"                  US	9u  nnUS:w  a  S
u  nnOU R%                  U5      u  nnU R                   R'                  US-   5      nXo-   nU R(                  R*                  nUS:w  a`  UR-                  S5      S:X  d   S5       e[/        U R0                  5       Vs/ s H  nUUS-
     U-   R3                  USS5      PM!     nnSnSnOR[/        U R0                  5       Vs/ s H  nUUS-
     U-   PM     nnU R5                  UU5      nU R7                  UU5      nUb}  SUSS2SSSS24   R3                  SU R                   R8                  SS5      -
  [:        R<                  " U R>                  5      R@                  -  nURC                  UR>                  5      nOSn[:        RD                  " U/U-   S5      nU RF                  (       a  U RG                  U5      n[H        RJ                  RM                  UU RL                  U R                  S9nU	(       a  SOSnU	(       a  U R                   R0                  S:  a  SOSnU(       a  SOSnU(       a  SOSnU(       a  U R                   RN                  (       a  SOSn[Q        U RR                  5       H  u  n n!U	(       a8  UUSS2SU24   4-  nU R                   R0                  S:  a  UUSS2US24   4-  nU!" UUUUUUUUUUUS9n"U"S   nU(       d  Mc  UU"S   4-  nUU"S   4-  nU R                   RN                  (       d  M  UU"S   4-  nM     U	(       a8  UUSS2SU24   4-  nU R                   R0                  S:  a  UUSS2US24   4-  nUSS2SU24   n#U R                   R0                  S:  a  USS2US24   OSn$U
(       d  [U        S U#U$UUUUUU4 5       5      $ [W        U#U$UUUUUUS9$ s  snf s  snf )a  
Example:

```python
>>> from transformers import AutoTokenizer, ProphetNetDecoder
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
>>> model = ProphetNetDecoder.from_pretrained("microsoft/prophetnet-large-uncased", add_cross_attention=False)
>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> last_hidden_states = outputs.last_hidden_state
```NzGEither `decoder_input_ids` or `decoder_inputs_embeds` has to be passed.zFMake sure to only pass `decoder_input_ids` or `decoder_inputs_embeds`.r&   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r   r   )r$   r\   )NNr   zOAt the moment `use_cache` is only supported for `decoder_input_ids` of length 1rx  r   re   )rT  r  rR   rS   rO   r\   rR  r   r   c              3   0   #    U  H  nUc  M  Uv   M     g 7fr   re   rz  s     r!   r}  ,ProphetNetDecoder.forward.<locals>.<genexpr>  s"      	A  	s   	)rr   rs   r\   rw   rx   ry   rz   ra   ),r   rR  r   ru  rv  r   r`  r   re  r   loggerwarning_onceis_encoder_decoderr   r
   r   ra  r$   !compute_buffered_relative_bucketsr   r  weightrN   r,   r1   rM   prepare_attention_maskprepare_predict_attention_maskr   r   r(   r   r)   r  r/   rb  r   r   r   rM  r  r   rm   ru   )%r   r   r   rc   r  r\   rt  rR  r   ru  rv  r   r   r0   past_key_values_lengthmain_stream_pos_embedrO   rR   rS   predicting_stream_pos_embedrw   r  r1   ngram_hidden_statesr  r  extended_encoder_attention_maskall_main_stream_hidden_statesall_ngram_stream_hidden_statesall_main_stream_attnsall_ngram_stream_attnsall_cross_attnsr  decoder_layerr  rr   rs   s%                                        r!   r   ProphetNetDecoder.forward7  s   : "+!6IDKK<Q<Q	1B1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY!6fgg"}'@eff"}'< 00;M&3&9&9"1&=#
&&4==##p "	0 )48V8V $L$DlZ^ZeZeFfg!5  FUE`!?!?!Afg.2.F.F) ''+ /G /
+| "Q&PZM*,M
 66|D.1&*&>&>&G&GWXHX&Y# &=0077 "Q& %%a(A- a- #4::.#.E "%!),/JJRRS]_`bcd.   # '+#.2+ Z__c_i_iYj#YjPU!%!),/JJYj   # '+&A&A-Q_&`#.2.Q.QR_ao.p+ "-,QdA-=>EEaIpIprsuvwwDJJ'++/,+ /N.P.PQ^QdQd.e+.2+		=/4G"GK%% 66}EM--mt||VZVcVc-d /C%/CHYHY\]H]cg&&7T'8d 1dkk6U6U"[_"+DKK"8C#--CSOCS@S2T1VV-;;$$q(2}QHXEX7Y6[[2)'%"A0O/M2S) /#"3M *!,M  %-*:)<<%&=+;*==&;;222#a(8'::O7 #9:  )mA?O?O<O.P-RR){{  1$.=ODTAT3U2WW. *!-=o-=*=>HLHYHY\]H]-?3C0C"Dcg  &+#12)*#	   ,/$;+7 >,3,	
 		
u##s   %&U)Uc           	         UR                   u  p#[        R                  " SU R                  5      R	                  UR
                  5      R                  SS5      n[        U R                  U R                  U5      u  pEUS S 2S U2S U24   R                  USS5      n[        R                  " US S 2S U2S U24   US S 2S U2U R                  U R                  U-   24   /S5      R                  USS5      nXE4$ r   )r   r   r0  r  r  r$   rM   rT   r@   r   r/   )r   rO   r   r0   main_relative_bucketspredict_relative_bucketss         r!   r  3ProphetNetDecoder.compute_buffered_relative_buckets  s   &2&8&8#
||At'@'@ADD\EXEXY``abdef:]d88,;
7
 !6a9I/9IK[OK[6[ \ c cdnpqst u#(99(,<_,<>N>N)NO('')B)BTE^E^apEp)pp $
 &Q
" 	! %>>r"   c                 H   UR                   S S u  p4[        R                  " XD4[        R                  " UR                  5      R
                  UR                  UR                  S9n[        R                  " US5      nUS U2S U24   S S S S 2S S 24   R                  X0R                  R                  4UR                   -   5      nUb@  SUS S 2S S S S 24   -
  [        R                  " U R                  5      R
                  -  nXg-   nOUnUR                  UR                  5      $ )Nr&   r   r   rx  )r   r   fullr(   r   r)   r$   triuexpandr   r   r  )r   rw   r   r   
seq_lengthcausal_maskextended_causal_maskr  s           r!   r  (ProphetNetDecoder.prepare_attention_mask	  s   !.!4!4Ra!8
 jj$KK++,00%% ''	
 jja0*;J;+CDT4QRTUEUV]]@@AKDUDUU 

 %'*^AtT1<L-M'MQVQ\Q\]a]g]gQhQlQl&l#&:&T#&:#&))-*=*=>>r"   c           	         UR                   S S u  p4[        U R                  U R                  UR                  UR
                  5      n[        R                  " US S 2S U2S U24   US S 2S U2U R                  U R                  U-   24   /SS9nUS S S S 2S S 2S S 24   R                  X0R                  R                  4UR                   -   5      nUb  SUS S 2S S S S S 24   -
  [        R                  " U R
                  5      R                  -  nUR                  X0R                  R                  U R                  XD45      n[        R                  " U[        R                  " U5      /SS9nXg-   nOUnUR                  UR
                  5      $ )Nr&   rK   r   rx  )r   r5   r  r1   r$   r   r   r/   r  r   r   r(   r)   r8   r  )	r   rw   r   r   r  predict_causal_maskextended_predict_causal_maskr  r  s	            r!   r  0ProphetNetDecoder.prepare_predict_attention_mask!  s   !.!4!4Ra!8
 3%%tzz=3G3GI\I\
 $ii#A{
{KZK$?@#{
{D$=$=@Y@Y\f@f$ff 
 (;4q!Q;N'O'V'V@@ADWD]D]](
$
 %'*^AtT4QR<R-S'SW\WbWbcgcmcmWnWrWr&r#&=&D&D[[DDdjjR\i'# ',ii(%*:*:;R*STZ\'# /K.d+.J+.11-2E2EFFr"   )r   rb  re  r   r  r1   r  r@   ra  r   r`  )
NNNNNNNNNN)rf   rg   rh   ri   r   r   rk  rq  r   r   r   r	   r   rm   ru   r   r  r  r  rn   r   r   s   @r!   r  r    s%   / ,$%  *..2596:(,-1!%)-,0#'y
<<$&y
 t+y
  %||d2	y

 !&t 3y
 y
 ||d*y
 $;y
  $;y
 #Tky
 D[y
 
-	-y
 y
v?,?0!G !Gr"   r  c                   p  ^  \ rS rSrSSS.rS\4U 4S jjrS rS r\	            SS	\
R                  S-  S
\
R                  S-  S\
R                  S-  S\
R                  S-  S\S-  S\S-  S\
R                  S-  S\
R                  S-  S\S-  S\S-  S\S-  S\S-  S\\-  4S jj5       rSrU =r$ )ProphetNetModeliE  word_embeddings.weight)zencoder.word_embeddings.weightdecoder.word_embeddings.weightr   c                 j  > [         TU ]  U5        [        R                  " UR                  UR
                  UR                  S9U l        [        R                  " U5      nSUl
        [        U5      U l        [        R                  " U5      nSUl        [        U5      U l        U R!                  5         g )Nr]  FT)r   r   r   r^  r_  r   r   r`  copydeepcopyrR  r[  encoder
is_decoderr  decoderrf  )r   r   encoder_configdecoder_configr   s       r!   r   ProphetNetModel.__init__L  s     !||F,=,=v?Q?Q_e_r_rsv.#( (8v.$(!(8 	r"   c                     U R                   $ r   rj  r  s    r!   rk  $ProphetNetModel.get_input_embeddings[  rm  r"   c                 |    Xl         U R                   U R                  l         U R                   U R                  l         g r   )r`  r  r  ro  s     r!   rq  $ProphetNetModel.set_input_embeddings^  s,    $'+';';$'+';';$r"   Nr   r   decoder_input_idsdecoder_attention_maskencoder_outputsr\   rt  decoder_inputs_embedsrR  r   ru  rv  r   c                 V   U	b  U	OU R                   R                  n	U
b  U
OU R                   R                  n
Ub  UOU R                   R                  nUb  UOU R                   R                  nUc  U R                  UUUU
UUS9nU R                  UUUS   UUUU
UU	US9
nU(       d  X-   $ [        UR                  UR                  UR                  UR                  UR                  UR                  UR                  UR                  UR                  UR                  UR                  S9$ )a1  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
    `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
    `past_key_values`).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

Example:

```python
>>> from transformers import AutoTokenizer, ProphetNetModel

>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
>>> model = ProphetNetModel.from_pretrained("microsoft/prophetnet-large-uncased")

>>> input_ids = tokenizer(
...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1
>>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)

>>> last_hidden_states = outputs.last_hidden_state  # main stream hidden states
>>> last_hidden_states_ngram = outputs.last_hidden_state_ngram  # predict hidden states
```)r   r   rt  r   ru  rv  r   )
r   r   rc   r  r\   rt  r   ru  rR  rv  )rr   rs   r\   r]   r^   r_   r`   ra   rb   rc   rd   )r   rR  r   ru  rv  r  r  rp   rr   rs   r\   rw   rx   ry   rz   ra   )r   r   r   r  r  r  r\   rt  r  rR  r   ru  rv  r   decoder_outputss                  r!   r   ProphetNetModel.forwardc  sI   d "+!6IDKK<Q<Q	1B1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY""ll#-+"3%9' + O ,,'1"1!"4#1+//!5# ' 
 "44+-??$3$K$K+;;"1"?"?(7(K(K.99%4%E%E,==&5&G&G"1"?"?.99
 	
r"   )r  r  r`  )NNNNNNNNNNNN)rf   rg   rh   ri   _tied_weights_keysr   r   rk  rq  r   r   r   
BoolTensorrm   r	   r   rp   r   rn   r   r   s   @r!   r  r  E  sJ    +C*B
/ $<
  *..215:>(,(,-159!%)-,0#'^
<<$&^
 t+^
 !<<$.	^

 !& 0 04 7^
 ^
 ^
 ||d*^
  %||d2^
 $;^
  $;^
 #Tk^
 D[^
 
-	-^
 ^
r"   r  zh
    The ProphetNet Model with a language modeling head. Can be used for sequence generation tasks.
    c                     ^  \ rS rSrSS0rS\4U 4S jjrS r\             SS\	R                  S-  S	\	R                  S-  S
\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S\S-  S\S-  S\S-  S\\-  4S jj5       rSS jrS\	R                  4S jrSU 4S jjrSrU =r$ )"ProphetNetForConditionalGenerationi  lm_head.weight!prophetnet.word_embeddings.weightr   c                   > [         TU ]  U5        [        U5      U l        UR                  U l        UR                  U l        [        R                  " UR                  UR                  SS9U l        U R                  5         g )NFbias)r   r   r  r   r   r   disable_ngram_lossr   r   r   r_  lm_headrf  r   s     r!   r   +ProphetNetForConditionalGeneration.__init__  sd     )&1!.."(";";yy!3!3V5F5FUS 	r"   c                 .    U R                   R                  $ r   )r   r`  r  s    r!   rk  7ProphetNetForConditionalGeneration.get_input_embeddings  s    ...r"   Nr   r   r  r  r  r\   rt  r  labelsrR  r   ru  rv  r   c                 r   Ub  UOU R                   R                  nU	b  Uc  Uc  U R                  U	5      nU R                  UUUUUUUUU
UUUS9nUb  UR                  OUR                  SS u  nnUS   R                  UU R                   R                  US5      nU R                  U5      nUSS2S4   nU R                   R                  S:  a  USS2SS24   OSnUR                  5       (       d  UR                  5       nSnU	b  U R                  UU	5      nU(       d+  [        S UU4 5       5      nUb  U4U-   USS -   $ UUSS -   $ [        UUUUR                  UR                  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  S9$ )	a  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
    `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
    `past_key_values`).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
    config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
    labels in `[0, ..., config.vocab_size]`

Example:

```python
>>> from transformers import AutoTokenizer, ProphetNetForConditionalGeneration

>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
>>> model = ProphetNetForConditionalGeneration.from_pretrained("microsoft/prophetnet-large-uncased")

>>> input_ids = tokenizer(
...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1
>>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)

>>> logits_next_token = outputs.logits  # logits to predict next token as usual
>>> logits_ngram_next_tokens = outputs.logits_ngram  # logits to predict 2nd, 3rd, ... next tokens
```N)r   r   r  r  r  r\   rt  r  rR  r   ru  rv  r&   r   rK   r   c              3   .   #    U  H  oc  M  Uv   M     g 7fr   re   rz  s     r!   r}  =ProphetNetForConditionalGeneration.forward.<locals>.<genexpr>;       R*@Qqq*@r  )rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   )r   rv  r   r   r   r   r1   r  is_contiguousr   _compute_lossrm   rW   r\   r]   r^   r_   r`   ra   rb   rc   rd   )r   r   r   r  r  r  r\   rt  r  r  rR  r   ru  rv  r   rE  r   r0   predicting_streamspredict_logitsrZ   r[   rY   
all_logitss                           r!   r   *ProphetNetForConditionalGeneration.forward  s   n &1%<k$++BYBY"3";@U@] $ 1 1& 9//)/#9++'"7/!5# " 
 (9'D##J_JeJefhghJi 	$
O %QZ__Z9J9JO]_`&891%040A0AA0E~ae,4 ##%%&&(F%%nf=DR6<*@RRJ9=9ID7Z''!"+5gz\cdedf\gOgg,) ' 7 7&-&C&C,3,O,O#*#=#=)0)I)I!(!9!9*1*K*K&-&C&C#*#=#= r"   c                    UR                  U R                  R                  UR                  S5      UR                  S5      5      R	                  U5      n[        U R                  R                  5       H'  nUS:  a  U R                  (       a    OX$US S 2S S 24'   M)     UR                  SS5      R                  5       n[        R                  R                  UR                  SUR                  S5      5      S[        R                  S9n[        R                  R                  XdR                  S5      SS9nU R                  R                   S:  a  UR#                  SSS	9* nUR%                  U5      R                  S5      n	X   nUR'                  5       nU R                  R                   UR                  S5      -  n
S
U R                  R                   -
  U-  X-  -   nU$ Nr   r   rK   r   mean)	reductiong        T)r   keepdimrx  r   r   r1   rN   fill_r,   r  r   r   r   r   log_softmaxr   r   r   nll_lossepssumner  r   rZ   r  ignore_indexexpend_targetsr  lprobsrY   smooth_lossnon_masked_tokenseps_is              r!   r  0ProphetNetForConditionalGeneration._compute_lossM     ))$++*;*;V[[^V[[YZ^\bbcopt{{(()A1u00&,1a7# *
 !!!Q'224**KKFKKO,-- + 
 }}%%f.A.A".EQW%X;;??S !::"d:;;K . 1 1, ? D DR H%8K%**,KKKOOfkk"o5E$++//)T1E4GGDr"   c                 $    U R                  U5      $ r   )r   )r   r  s     r!   %prepare_decoder_input_ids_from_labelsHProphetNetForConditionalGeneration.prepare_decoder_input_ids_from_labelsi  s      ((r"   c                 P   > Uc  U R                   R                  $ [        TU ]  US9$ )N)modality)r   r  r   get_encoder)r   r  r   s     r!   r  .ProphetNetForConditionalGeneration.get_encoderl  s,    ??***7&&99r"   r  r  r   r   )NNNNNNNNNNNNNr   r   )rf   rg   rh   ri   r  r   r   rk  r   r   r   r  r	   r   rm   rW   r   r  r  r  rn   r   r   s   @r!   r  r    s    	=	/ 	/  *..215:>/3(,-159&*!%)-,0#'m<<$&m t+m !<<$.	m
 !& 0 04 7m ,m m ||d*m  %||d2m t#m $;m  $;m #Tkm D[m  
*	*!m m^8)ELL ): :r"   r  zt
    The standalone decoder part of the ProphetNetModel with a lm head on top. The model can be used for causal
    c                   n  ^  \ rS rSrSSS.rS\4U 4S jjrS rS r\	           SS	\
R                  S-  S
\
R                  S-  S\
R                  S-  S\
R                  S-  S\S-  S\
R                  S-  S\
R                  S-  S\S-  S\S-  S\S-  S\S-  S\\-  4S jj5       rSS jrSrU =r$ )ProphetNetForCausalLMis  r  )r  z)prophetnet.decoder.word_embeddings.weightr   c                 N  > [         R                  " U5      nSUl        SUl        [        TU ]  U5        [        U5      U l        UR                  U l	        UR                  U l
        [        R                  " UR                  UR                  SS9U l        U R!                  5         g )NTFr  )r  r  r  r  r   r   ProphetNetDecoderWrapperr   r   r   r  r   r   r   r_  r  rf  r   s     r!   r   ProphetNetForCausalLM.__init__~  s    v& $)! 26:!.."(";";yy!3!3V5F5FUS 	r"   c                 B    U R                   R                  R                  $ r   r   r  r`  r  s    r!   rk  *ProphetNetForCausalLM.get_input_embeddings  s    &&666r"   c                 8    XR                   R                  l        g r   r  ro  s     r!   rq  *ProphetNetForCausalLM.set_input_embeddings  s    27/r"   Nr   r   rc   r  r\   rt  r  rR  r   ru  rv  r   c                    Ub  UOU R                   R                  nU R                  R                  UUUUUUUU	U
US9
nUb  UR                  OUR                  SS u  pUS   R                  XR                   R                  US5      nU R                  U5      nUSS2S4   nU R                   R                  S:  a  USS2SS24   OSnSnUb  U R                  UU5      nU(       d+  [        S UU4 5       5      nUb  U4U-   USS -   $ UUSS -   $ [        UUUUR                  UR                  UR                  UR                  UR                  UR                   S9	$ )	a  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
    `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
    ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`

Example:

```python
>>> from transformers import AutoTokenizer, ProphetNetForCausalLM
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
>>> model = ProphetNetForCausalLM.from_pretrained("microsoft/prophetnet-large-uncased")
>>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder."
>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> logits = outputs.logits

>>> # Model can also be used with EncoderDecoder framework
>>> from transformers import BertTokenizer, EncoderDecoderModel, AutoTokenizer
>>> import torch

>>> tokenizer_enc = BertTokenizer.from_pretrained("google-bert/bert-large-uncased")
>>> tokenizer_dec = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
>>> model = EncoderDecoderModel.from_encoder_decoder_pretrained(
...     "google-bert/bert-large-uncased", "microsoft/prophetnet-large-uncased"
... )

>>> ARTICLE = (
...     "the us state department said wednesday it had received no "
...     "formal word from bolivia that it was expelling the us ambassador there "
...     "but said the charges made against him are `` baseless ."
... )
>>> input_ids = tokenizer_enc(ARTICLE, return_tensors="pt").input_ids
>>> labels = tokenizer_dec(
...     "us rejects charges against its ambassador in bolivia", return_tensors="pt"
... ).input_ids
>>> outputs = model(input_ids=input_ids, decoder_input_ids=labels[:, :-1], labels=labels[:, 1:])

>>> loss = outputs.loss
```N)
r   r   rc   r  r\   rt  rR  r   ru  rv  r&   r   rK   r   c              3   .   #    U  H  oc  M  Uv   M     g 7fr   re   rz  s     r!   r}  0ProphetNetForCausalLM.forward.<locals>.<genexpr>  r  r  )	rY   rZ   r[   r\   rw   rx   ry   rz   ra   )r   rv  r   r  r   r   r1   r  r  rm   r|   r\   rw   rx   ry   rz   ra   )r   r   r   rc   r  r\   rt  r  rR  r   ru  rv  r   rE  r   r0   r  r  rZ   r[   rY   r  s                         r!   r   ProphetNetForCausalLM.forward  s   v &1%<k$++BYBY //)))"7#9+'/!5# * 
 :C9NiooTaTgTghjijTk#
$QZ__Z9J9JO]_`&891%040A0AA0E~ae,4%%nf=DR6<*@RRJ9=9ID7Z''!"+5gz\cdedf\gOgg,) ' 7 7%33$+$?$?"--!(!9!9!(!9!9
 
r"   c                    UR                  U R                  R                  UR                  S5      UR                  S5      5      R	                  U5      n[        U R                  R                  5       H'  nUS:  a  U R                  (       a    OX$US S 2S S 24'   M)     UR                  SS5      R                  5       n[        R                  R                  UR                  SUR                  S5      5      S[        R                  S9n[        R                  R                  XdR                  S5      SS9nU R                  R                   S:  a  UR#                  SSS	9* nUR%                  U5      R                  S5      n	X   nUR'                  5       nU R                  R                   UR                  S5      -  n
S
U R                  R                   -
  U-  X-  -   nU$ r  r  r  s              r!   r  #ProphetNetForCausalLM._compute_loss  r  r"   r  )NNNNNNNNNNNr  )rf   rg   rh   ri   r  r   r   rk  rq  r   r   r   r	   r   rm   r|   r   r  rn   r   r   s   @r!   r
  r
  s  s@    >5X
/  78  *..2596:(,-1&*!%)-,0#'d<<$&d t+d  %||d2	d
 !&t 3d d ||d*d t#d $;d  $;d #Tkd D[d 
*	*d dL r"   r
  c                   B   ^  \ rS rSrSrSS0rS\4U 4S jjrS rSr	U =r
$ )	r  i  zx
This is a wrapper class, so that [`ProphetNetForCausalLM`] can correctly be loaded from pretrained prophetnet
classes.
r  r  r   c                    > [         TU ]  U5        [        R                  " UR                  UR
                  UR                  S9U l        [        U5      U l	        U R                  5         g )Nr]  )r   r   r   r^  r_  r   r   r`  r  r  rf  r   s     r!   r   !ProphetNetDecoderWrapper.__init__"  sP     !||F,=,=v?Q?Q_e_r_rs(0 	r"   c                 &    U R                   " U0 UD6$ r   )r  )r   argsr   s      r!   r    ProphetNetDecoderWrapper.forward+  s    ||T,V,,r"   )r  r`  )rf   rg   rh   ri   rj   r  r   r   r   rn   r   r   s   @r!   r  r    s/     	)*B/ - -r"   r  )r  r[  r
  r  r  r   rG  )9rj   r  r=   dataclassesr   r   r   r   torch.nnr   activationsr   cache_utilsr	   r
   r   
generationr   modeling_layersr   modeling_outputsr   modeling_utilsr   utilsr   r   r   configuration_prophetnetr   
get_loggerrf   r  r   r5   rI   rT   rW   rp   ru   r|   r   r^  r   Moduler   r   r   r9  rJ  r[  r  r  r  r
  r  __all__re   r"   r!   <module>r,     s   Y   !    ! C C ) 9 / - 9 9 6 
		H	%Q7" 6M. 
 *? *? *?Z  (?; (? (?V 
 #=; #= #=L 
 += += +=\ ! ! !8(-R\\ (-Vs2")) s2lBII .n/299 n/b	&7 &RD7 DN 
c
1 c

c
L 
hG1 hG
hGV	 |
/ |
 |
~ 
f:)BO f:
f:R 
]5 ]
]@-8 -.r"   